{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# MISCELLANEOUS IMPORTS\n",
    "import pandas as pd\n",
    "import patsy\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "\n",
    "# MODELLING AND METRICS IMPORTS\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier, PassiveAggressiveClassifier, Perceptron\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# SET SEABORN STYLE\n",
    "sns.set(context='paper', style='white', palette='bone', font='sansserif', font_scale=1.5, color_codes=False, rc=None)\n",
    "\n",
    "# SET FONT DICTIONARY\n",
    "font = {'family': 'sansserif',\n",
    "        'color':  'black',\n",
    "        'weight': 'normal',\n",
    "        'variant': 'small-caps',\n",
    "        'size': 14\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# DATA CLEANING\n",
    "data_raw = pd.read_csv('german_credit_risk.csv', na_values = ['.'])\n",
    "data = data_raw.ix[:,1:10]\n",
    "label = data_raw.ix[:,10]\n",
    "datan = data\n",
    "datan = datan.replace(['male','female'],[0,1])\n",
    "datan = datan.replace(['free','rent','own'],[0,1,2])\n",
    "datan = datan.replace(['little','moderate','quite rich','rich'],[1,2,3,4])\n",
    "datan = datan.replace(['radio/TV','repairs','domestic appliances','vacation/others','furniture/equipment','car','education','business'],[1,2,3,4,5,6,7,8])\n",
    "datan = datan.fillna(0)\n",
    "enc = OneHotEncoder(categorical_features = [1,2,3,4,5,8],sparse = False)\n",
    "enc.fit(datan)\n",
    "dataprocessed = enc.transform(datan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('german_credit_risk.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Sex'] = [0 if x == 'male' else 1 for x in df['Sex']]\n",
    "df.fillna('no_facility', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Job</th>\n",
       "      <th>Housing</th>\n",
       "      <th>Saving accounts</th>\n",
       "      <th>Checking account</th>\n",
       "      <th>Credit amount</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Purpose</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>67</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>own</td>\n",
       "      <td>no_facility</td>\n",
       "      <td>little</td>\n",
       "      <td>1169</td>\n",
       "      <td>6</td>\n",
       "      <td>radio/TV</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>own</td>\n",
       "      <td>little</td>\n",
       "      <td>moderate</td>\n",
       "      <td>5951</td>\n",
       "      <td>48</td>\n",
       "      <td>radio/TV</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>own</td>\n",
       "      <td>little</td>\n",
       "      <td>no_facility</td>\n",
       "      <td>2096</td>\n",
       "      <td>12</td>\n",
       "      <td>education</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>free</td>\n",
       "      <td>little</td>\n",
       "      <td>little</td>\n",
       "      <td>7882</td>\n",
       "      <td>42</td>\n",
       "      <td>furniture/equipment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>53</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>free</td>\n",
       "      <td>little</td>\n",
       "      <td>little</td>\n",
       "      <td>4870</td>\n",
       "      <td>24</td>\n",
       "      <td>car</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age  Sex  Job Housing Saving accounts Checking account  Credit amount  \\\n",
       "0   67    0    2     own     no_facility           little           1169   \n",
       "1   22    1    2     own          little         moderate           5951   \n",
       "2   49    0    1     own          little      no_facility           2096   \n",
       "3   45    0    2    free          little           little           7882   \n",
       "4   53    0    2    free          little           little           4870   \n",
       "\n",
       "   Duration              Purpose  Class  \n",
       "0         6             radio/TV      1  \n",
       "1        48             radio/TV      2  \n",
       "2        12            education      1  \n",
       "3        42  furniture/equipment      1  \n",
       "4        24                  car      2  "
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import patsy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'Age', u'Sex', u'Job', u'Housing', u'Saving accounts',\n",
       "       u'Checking account', u'Credit amount', u'Duration', u'Purpose',\n",
       "       u'Class'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "formula = 'Class~Age+Sex+C(Job)-1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "y, X = patsy.dmatrices(formula, data=df, return_type='dataframe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>C(Job)[0]</th>\n",
       "      <th>C(Job)[1]</th>\n",
       "      <th>C(Job)[2]</th>\n",
       "      <th>C(Job)[3]</th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>970</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>971</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>972</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>973</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>974</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>975</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>976</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>977</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>978</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>979</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>981</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>982</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>983</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>984</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>985</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>986</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>990</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>991</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     C(Job)[0]  C(Job)[1]  C(Job)[2]  C(Job)[3]   Age  Sex\n",
       "0          0.0        0.0        1.0        0.0  67.0  0.0\n",
       "1          0.0        0.0        1.0        0.0  22.0  1.0\n",
       "2          0.0        1.0        0.0        0.0  49.0  0.0\n",
       "3          0.0        0.0        1.0        0.0  45.0  0.0\n",
       "4          0.0        0.0        1.0        0.0  53.0  0.0\n",
       "5          0.0        1.0        0.0        0.0  35.0  0.0\n",
       "6          0.0        0.0        1.0        0.0  53.0  0.0\n",
       "7          0.0        0.0        0.0        1.0  35.0  0.0\n",
       "8          0.0        1.0        0.0        0.0  61.0  0.0\n",
       "9          0.0        0.0        0.0        1.0  28.0  0.0\n",
       "10         0.0        0.0        1.0        0.0  25.0  1.0\n",
       "11         0.0        0.0        1.0        0.0  24.0  1.0\n",
       "12         0.0        0.0        1.0        0.0  22.0  1.0\n",
       "13         0.0        1.0        0.0        0.0  60.0  0.0\n",
       "14         0.0        0.0        1.0        0.0  28.0  1.0\n",
       "15         0.0        1.0        0.0        0.0  32.0  1.0\n",
       "16         0.0        0.0        1.0        0.0  53.0  0.0\n",
       "17         0.0        0.0        1.0        0.0  25.0  0.0\n",
       "18         0.0        0.0        0.0        1.0  44.0  1.0\n",
       "19         0.0        0.0        1.0        0.0  31.0  0.0\n",
       "20         0.0        0.0        1.0        0.0  48.0  0.0\n",
       "21         0.0        0.0        1.0        0.0  44.0  0.0\n",
       "22         0.0        1.0        0.0        0.0  48.0  0.0\n",
       "23         0.0        0.0        1.0        0.0  44.0  0.0\n",
       "24         0.0        0.0        1.0        0.0  26.0  0.0\n",
       "25         0.0        1.0        0.0        0.0  36.0  0.0\n",
       "26         0.0        1.0        0.0        0.0  39.0  0.0\n",
       "27         0.0        0.0        1.0        0.0  42.0  1.0\n",
       "28         0.0        0.0        1.0        0.0  34.0  0.0\n",
       "29         0.0        0.0        1.0        0.0  63.0  0.0\n",
       "..         ...        ...        ...        ...   ...  ...\n",
       "970        0.0        0.0        1.0        0.0  22.0  0.0\n",
       "971        0.0        1.0        0.0        0.0  43.0  0.0\n",
       "972        1.0        0.0        0.0        0.0  29.0  1.0\n",
       "973        0.0        0.0        1.0        0.0  36.0  0.0\n",
       "974        0.0        0.0        1.0        0.0  33.0  1.0\n",
       "975        0.0        1.0        0.0        0.0  57.0  1.0\n",
       "976        0.0        0.0        1.0        0.0  64.0  1.0\n",
       "977        0.0        0.0        1.0        0.0  42.0  0.0\n",
       "978        0.0        1.0        0.0        0.0  47.0  0.0\n",
       "979        0.0        0.0        1.0        0.0  25.0  0.0\n",
       "980        0.0        0.0        1.0        0.0  49.0  0.0\n",
       "981        0.0        0.0        0.0        1.0  33.0  0.0\n",
       "982        0.0        0.0        0.0        1.0  28.0  1.0\n",
       "983        0.0        0.0        1.0        0.0  26.0  0.0\n",
       "984        0.0        1.0        0.0        0.0  30.0  0.0\n",
       "985        0.0        0.0        1.0        0.0  25.0  1.0\n",
       "986        0.0        0.0        1.0        0.0  33.0  0.0\n",
       "987        0.0        0.0        1.0        0.0  64.0  1.0\n",
       "988        0.0        0.0        0.0        1.0  29.0  0.0\n",
       "989        0.0        1.0        0.0        0.0  48.0  0.0\n",
       "990        0.0        1.0        0.0        0.0  37.0  0.0\n",
       "991        0.0        1.0        0.0        0.0  34.0  0.0\n",
       "992        0.0        1.0        0.0        0.0  23.0  0.0\n",
       "993        0.0        0.0        0.0        1.0  30.0  0.0\n",
       "994        0.0        0.0        1.0        0.0  50.0  0.0\n",
       "995        0.0        1.0        0.0        0.0  31.0  1.0\n",
       "996        0.0        0.0        0.0        1.0  40.0  0.0\n",
       "997        0.0        0.0        1.0        0.0  38.0  0.0\n",
       "998        0.0        0.0        1.0        0.0  23.0  0.0\n",
       "999        0.0        0.0        1.0        0.0  27.0  0.0\n",
       "\n",
       "[1000 rows x 6 columns]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TRAIN AND TEST SPLIT WITH TEST SIZE OF 20%\n",
    "data_train,data_test,label_train,label_test = train_test_split(dataprocessed,label,test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes, title='Confusion matrix', cmap=plt.cm.bone):\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title, fontdict=font)\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"black\" if cm[i, j] > thresh else \"white\",\n",
    "                fontdict=font)\n",
    "    plt.ylabel('True label', fontdict=font, fontsize=10)\n",
    "    plt.xlabel('Predicted label', fontdict=font, fontsize=10)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_parameters = {\n",
    "           \"clf__penalty\"      : ['l2'],\n",
    "           \"clf__C\"            : [0.001, 0.01, 0.1, 1, 100],\n",
    "           \"clf__solver\"       : ['liblinear', 'sag'],\n",
    "           \"clf__tol\"          : [0.0001, 0.001, 0.01],\n",
    "#            \"pca__n_components\" : [2, 8, 10, 20]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 30 candidates, totalling 150 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 150 out of 150 | elapsed:    3.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Gridsearch\n",
      "Grid Search Best Score: 0.71875\n",
      "Grid Search Best Params: {'clf__tol': 0.01, 'clf__penalty': 'l2', 'clf__C': 0.01, 'clf__solver': 'sag'}\n",
      "Grid Search Best Estimator: Pipeline(steps=[('ss', StandardScaler(copy=True, with_mean=True, with_std=True)), ('pca', PCA(copy=True, iterated_power='auto', n_components=None, random_state=42,\n",
      "  svd_solver='auto', tol=0.0, whiten=False)), ('clf', LogisticRegression(C=0.01, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='sag', tol=0.01,\n",
      "          verbose=0, warm_start=False))])\n",
      "Classification Report:              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.74      0.94      0.83       138\n",
      "          2       0.68      0.27      0.39        62\n",
      "\n",
      "avg / total       0.72      0.73      0.69       200\n",
      "\n",
      "Accuracy: 0.735\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAAEvCAYAAACe8d6ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHjtJREFUeJzt3X1czXfjP/DX6XRKnRKtJLV1EXKTbthGW7tMWJthxcbY\n3FzMdbm6cWEzN127mLu12djGhNy0qcXPZv3EuMKIMJJirZjijEJyE0J357y/f/g6X2dJmd7n6PR6\nPh7nofN5f87n8zqHXt6fz7lTCCEEiIgksDB1ACIyXywYIpKGBUNE0rBgiEgaFgwRScOCISJpWDCN\nyKxZs+Di4gKFQoG4uLhH3p5Go4FCocDhw4cfPVwD8OKLLyIiIsLUMRoUBV8HY1pFRUWYP38+Nm/e\njIKCAjg5OcHHxweRkZHo169fve0nOzsbXbp0wcaNGxEQEAAHBwfY2Ng80ja1Wi2Ki4vh5OQES0vL\nekpqPBqNBq1bt0Z6ejqefvrpWte/cuUKVCoV7O3tjZDOPDS8fxVmRKPR4Pnnn4e9vT0++ugj+Pr6\nQqfTYefOnRg/fjzOnDlTb/vKy8sDAISEhEChUNTLNpVKJVq2bFkv23qcVVRUwMrKCo6OjqaO0vAI\nMplXXnlFuLq6ihs3blQbu3Lliv7n33//XYSEhAg7OzthZ2cnQkNDxdmzZ/XjM2fOFJ07dxaJiYmi\nTZs2ws7OTrz22muiuLhYPw7A4CKEEKNGjRKvvvqqwX7vbuuuY8eOiaCgIGFvby/s7OyEj4+P+Omn\nn4QQQpw+fVoAEOnp6fr1U1NTxbPPPiusra1FixYtxMSJE0V5ebl+vGfPnuKf//ynmD59unjiiSeE\ns7OzePfdd4VWq63xcVqzZo1Qq9Xixx9/FF5eXsLGxkYMGDBAlJSUiA0bNoi2bduKpk2birffflvc\nunVLf7utW7eKwMBA0axZM9G8eXPx0ksviZycHP34Hx+Tnj17Gjwu0dHRws3NTTg7O+uzh4eHCyGE\nOH78uLC1tRVff/21wf5UKpU4cOBAjfelseE5GBO5cuUKtm3bhoiICNjZ2VUbb968OQBACIGQkBAU\nFRXhp59+wq5du3Du3DmEhIRA3HN0q9FosH79evzwww9ISUlBZmYmoqKiAADvvfceYmNjAQDnz5/H\n+fPn65xz+PDhcHV1xaFDh5CZmYlZs2ahSZMm9123sLAQr7zyCvz9/ZGZmYlVq1YhMTER06dPN1gv\nISEBlpaW2L9/P5YsWYLPP/8c69evf2CO8vJyfPbZZ0hISMDOnTtx+PBhvP766/j666/x/fffIykp\nCZs3b8bSpUv1t7l58yYmTpyIQ4cOYffu3XBwcMCAAQNQUVEBADh06BAAYNu2bTh//jw2btyov21q\naiqOHTuGbdu2YefOndXyeHl5YdGiRYiIiEB+fj6Ki4sxevRoREVFoUePHnV7cBsDUzdcY3Xw4EEB\nQGzcuPGB66WkpAgLCwtx+vRp/bL8/HyhUCjE9u3bhRB3Zh3W1taipKREv87cuXOFp6en/vqGDRvE\nH/+66zKDsbe3F3FxcffN9scZzIwZM4Snp6fBbGTNmjXCyspK3Lx5UwhxZxbQo0cPg+306dNHjB07\ntsbHYM2aNQKAOH78uH7Zu+++KywsLPSztJruz71KS0uFhYWF2Lt3733z37sdJycnUVZWZrD83hnM\nXSEhIaJ79+6iX79+4rnnnhNVVVU17r8x4gzGREQdz63n5uaiVatW+Mtf/qJf1qZNG7Rq1Qo5OTn6\nZR4eHnBwcNBfb9WqFS5evPjIOSdPnox33nkHQUFBmDdvHo4fP/7ArAEBAbCw+L9/VoGBgaioqNCf\nAwIAHx8fg9vVJau1tTW8vLz0111cXNCyZUs4OTkZLLt3O/n5+Rg+fDg8PT3RtGlTuLi4QKfT1enc\nlre3N6ytrWtdb+XKlcjLy8OePXsQHx8PpVJZ620aExaMibRr1w4KhQK5ubkPXE8IUeNJ2XuXq1Sq\namM6ne6B27awsKhWdJWVlQbXZ82ahZycHISEhGD//v3w8fHB6tWrjZ71j89SKRSKWrczYMAAFBcX\nY/ny5Th48CAyMzNhaWmpP0R6ELVaXes6wJ1n565du4aysjIUFhbW6TaNCQvGRBwdHREcHIwlS5ag\ntLS02nhJSQkAoFOnTigsLIRGo9GPnTp1CufOnUOnTp0eKYOzs3O18zFZWVnV1mvXrh0mTJiALVu2\nYOzYsVi5cuV9t9epUyccOHDA4Jc8LS0NVlZW8PT0fKSsD+vy5cvIzc3FjBkz0KdPH3Ts2BE3btxA\nVVWVfh0rKysAd55u/zNKSkowcuRIvPfeewgPD8eIESNw/fr1eslvLlgwJrR06VIIIfD0009jw4YN\nOHHiBI4fP46YmBj9YUSfPn3g6+uLt956CxkZGTh8+DDeeustdO3aFUFBQY+0/6CgIGRmZmL16tXI\ny8vDJ598gn379unHb9++jfDwcOzevRsajQYHDx5EWlpajcUWFhaGc+fOISwsDLm5udiyZQumTZuG\niIgI2NraPlLWh9W8eXM4OTkhNjYWeXl5SE1Nxfjx4w1mQi1atICNjQ3++9//oqioCNeuXXuofYwf\nPx5OTk6YPXs2Pv74Y9jb2yM8PLy+70qDxoIxodatW+PIkSPo27cvpk6dCh8fHwQFBWHTpk1Yvnw5\ngDvT/qSkJDg7O+PFF19Er1690LJlSyQlJT3y61mCg4Mxc+ZMREVFoVu3btBoNAgLC9OPK5VKXL16\nFaNGjYKXlxdCQ0MREBCAhQsX3nd7bm5u2Lp1KzIzM+Hn54cxY8Zg2LBhmD9//iPl/DMsLCywfv16\nHDt2DN7e3ggPD8ecOXMMzqtYWlriyy+/xMqVK9GqVSu89tprdd7+2rVrsWnTJiQkJEClUsHa2hrf\nfvstvvvuOyQmJsq4Sw0SX8lLRNJwBkNE0rBgiEgaFgwRScOCISJpHot3U5eVlSE7OxvOzs58JSRR\nA3P3Yzu8vb2rvU/tsSiY7OxsvPXWW6aOQUSPICEhodrn6jwWBePs7AzgTsDG8PkijVHr1q1NHYEk\nsbS0xFNPPaX/PTYYM0Geau4eFrVs2RLu7u4mTkMy3PsSfTJP9zu9wZO8RCQNC4aIpGHBEJE0LBgi\nkoYFQ0TSsGCISBoWDBFJw4IhImlYMEQkDQuGiKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJp\nWDBEJA0LhoikYcEQkTQsGCKShgVDRNKwYIhIGhYMEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgikoYF\nQ0TSsGCISBoWDBFJw4IhImlYMEQkDQuGiKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBE\nJA0LhoikYcEQkTQsGCKShgVDRNKwYIhIGhYMEUnDgqlHe/bswcCBA+Hm5gaFQoG4uDiD8Q8++AAd\nOnSAWq1G8+bN0bt3b+zfv99gnfLyckRGRsLJyQlqtRoDBw5EQUGBEe8F/RkWFhaYPXs2Tp06hdu3\nb+PUqVOYM2cOlEqlqaOZFAumHpWWlsLb2xtffPEFbGxsqo17eXnhq6++wi+//IK0tDS0bt0aL7/8\nMoqKivTrTJw4Ed9//z0SExOxd+9eXL9+Hf3794dWqzXmXaGHNHXqVISHh2PChAno0KED/vWvfyE8\nPBzTp083dTTTEo+Bs2fPivbt24uzZ8+aOkq9UavVYs2aNQ9c59q1awKA2LZtmxBCiJKSEqFSqUR8\nfLx+nTNnzgiFQqFfp6ECYNaX5ORkERcXZ7AsLi5OJCcnmzyb7IulpWWNv7+cwZhIRUUFVqxYgaZN\nm8LPzw8AkJGRgcrKSrz00kv69Z588kl07Nix2qEUPV7S0tLQq1cveHl5AQA6duyIoKAg/PjjjyZO\nZlqWpg7Q2GzevBlvvvkmbt26BVdXV2zfvh0uLi4AgAsXLkCpVMLJycngNi4uLrhw4YIp4lIdffzx\nx7C3t0dOTg60Wi1UKhXmzp2LmJgYU0czKc5gjKxXr17IysrC/v378fLLL2PIkCE4f/78A28jhIBC\noTBSQvozhg4dipEjR2L48OHo2rUrRowYgbCwMIwZM8bU0UxKasHEx8dj0KBB8Pb2xrRp02TuqsFQ\nq9Vo27YtevTogVWrVkGlUmHlypUAgJYtW0Kr1eLSpUsGt7l48aJ+lkOPpwULFuDTTz/F+vXrkZ2d\njfj4eCxcuLDRn+SVWjAtWrRAWFgYBg8eLHM3DZpOp0N5eTkAoFu3blCpVNi+fbt+vKCgALm5uXju\nuedMFZHqwNbWttozfVqtFhYWjfsgQeo5mLsnK3/55ReDp2LNVWlpKfLy8gDcKY4zZ84gKysLjo6O\naNasGT755BMMGDAArq6uKC4uxldffYWCggIMGTIEAODg4ICxY8diypQpaNGiBZ544glMnjwZPj4+\n6NOnjynvGtUiOTkZ06ZNw+nTp/Hrr7/C398fkydPxjfffGPqaKZljKcoFy5cKKZOnVrjuLk8Tb1r\n1677Po03atQocfPmTRESEiJcXV2FlZWVcHV1FQMHDhQ///yzwTZu374tIiIihKOjo7CxsRH9+/cX\nZ86cMdE9qj/3e1zM6WJnZycWLVokNBqNuHXrlsjPzxfz5s0T1tbWJs8m+/Kgp6kV//uXL9WiRYtQ\nVFSE6Ojo+44XFBSgd+/e2LlzJ9zd3WXHIRPgSWrzZWlpiTZt2tz397dxHyASkVQsGCKSRupJ3qqq\nKmi1Wuh0Omi1WpSXl0OpVMLSkq/vI2oMpM5gYmJi4OPjgxUrVmDTpk3w8fFp9K9sJGpMpE4lIiMj\nERkZKXMXRPQY4zkYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVDRNKw\nYIhIGhYMEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4IhImlYMEQkDQuG\niKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJI1lTQPXr19/4A2bNm1a72GIyLzUWDCd\nO3eGQqGAEEK/7O51hUKBM2fOGCUgETVcNRbM2bNnjZmDiMxQnc7BrFu3DvPnzwcAFBQUICMjQ2oo\nIjIPtRZMREQEdu3ahbVr1wIAbG1tMX78eOnBiKjhq/EQ6a79+/fjyJEj8Pf3BwA4OjqioqJCejAi\navhqncGoVCrodDooFAoAwOXLl2FhwWe3iah2tTZFeHg4Bg8ejOLiYsycOROBgYGYOnWqMbIRUQNX\n6yHSyJEj0a1bN+zYsQMAsGHDBnh7e0sPRkQNX60FAwBarRYqlQoKhQI6nU52JiIyE7UeIs2bNw/D\nhg3DuXPnUFBQgOHDh+Ojjz4yRjYiauBqncHEx8cjIyMDtra2AICoqCh069YN06dPlx6OiBq2Wmcw\nHh4eqKqq0l+vqqpCmzZtpIYiIvNQ4wxm0qRJUCgUsLW1RefOnREcHAyFQoGUlBQEBgYaMyMRNVA1\nFszdZ4o6d+6MV199Vb+8R48e8lMRkVmosWDGjh1rzBxEZIZqPcmbn5+PqKgo5OTkoKysTL/8t99+\nkxqMiBq+Wk/yjh49Gn/7298ghMDWrVsxZMgQvPnmm8bIRkQNXK0Fc+vWLQQHBwMAPD09MXfuXOza\ntUt6MCJq+Go9RLK2toYQAp6enli2bBnc3Nxw8eJFY2Qjogau1oJZtGgRSktL8eWXXyIqKgrXrl3D\n6tWrjZGNiBq4Wgume/fuAAB7e3v9h04REdVFjQUTGhqq/wyY+9m4caOUQERkPmosmIiICGPmAAC8\n/fa7sLS0Nvp+ST4HB2dTRyBJHvQBdDUWTO/evaWEIaLGg599SUTSsGCISJo6F0x5ebnMHERkhmot\nmEOHDqFLly5o164dAODo0aOIjIyUHoyIGr5aC2bChAnYvHkznnjiCQCAr68v3ypARHVSa8HodDp4\neHgYLFMqldICEZH5qPWVvE8++SQOHToEhUIBrVaLxYsXo3379sbIRkQNXK0zmJiYGCxcuBBnzpyB\ni4sLfv75Z8TExBgjGxE1cLXOYFq0aIF169YZIwsRmZlaC2bcuHH3fU/SihUrpAQiIvNRa8H06dNH\n/3NZWRl++OEHPPnkk1JDEZF5qLVghg4danB9xIgR6Nu3r7RARGQ+HvqtAqdPn8bvv/8uIwsRmZla\nZzDNmzfXn4PR6XRwdHREdHS09GBE1PA9sGCEEDh69Cjc3NwA3Pnchwd9CBUR0b0eeIikUCgQGhoK\npVIJpVLJciGih1LrOZhnn30WR44cMUYWIjIzNR4iVVVVwdLSEmlpaYiNjYWnpyfUajWEEFAoFCwd\nIqpVjQVzd+aSlJRkzDxEZEZqLBghBIA73+ZIRPRn1FgwxcXFWLhwYY03nDx5spRARGQ+aiwYrVaL\n0tJS/UyGiOhh1Vgwrq6u+M9//mPMLERkZmp8mpozFyJ6VDUWzM6dO42Zg4jMUI0F4+joaMwcRGSG\n+MVrRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4IhImlYMEQkDQuGiKRhwRCRNCwYIpKG\nBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVDRNKwYIhIGhYMEUnDgjGSYcMG\nYMeObxARMUK/bMqUcdix4xuDy+LF/LrehuC553ogMfEb5OQcRUnJRQwfPtRgvKTk4n0vCxZEmyix\nadT43dRUfzp29ES/fi8iP/9MtbGMjGxERy/XX6+qqjJmNPqT1Go1cnKOIzHx/2HZsiXVxtu39za4\n7u/vi/XrE5CU9P+NFfGxwIKRTK22wfTp4/HZZysxYkRItfHKyipcvXrNBMnoUWzfvhPbt9/5euWl\nSxdXG7948aLB9X79XsbJk3nYt++AUfI9LniIJNmkSWOwZ086srJy7zvu7d0OGzYsQVzcJ5g8eQya\nNbM3ckKSTa1WY9CgUHz9dbypoxgdZzAS9ev3Ilq1aoHo6GX3HU9PP4a0tMO4cKEYLi5O+NvfXseC\nBdMRFvYfVFbyUMlcvP76IFhbWyExcb2poxgdC0YSd/eWGDPmdUyaNA9VVdr7rrN790H9z6dPF+Dk\nSQ0SEhaie3c/pKUdNlZUkmzUqLexZctWXL582dRRjE5awVRUVGDWrFk4cOAASkpK4OHhgUmTJqFn\nz56ydvlY6dSpLZo1a4qVK+frlymVSnTp4oUBA4LQv/+4arOUy5dLUFx8FW5uLsaOS5J06eKNrl39\nMXv2/NpXNkPSCqaqqgqurq5Yu3YtWrVqhdTUVEycOBHJyclwd3eXtdvHxr59R/DOO9MNlk2ZMg6F\nhUX49ttN9z0EatrUDk5OzXHlSomxYpJko0aNwO+//47du1NNHcUkpBWMra0tIiMj9dd79eoFd3d3\n/Prrr42iYG7evIWbN28ZLCsrK8f166XQaArRpIk1Ro4Mxd69h3HlSglatnTC2LFDUFJyHWlpGSZK\nTXWlVqvRpk1rAICFhQLu7u7o0sUbV69eRUFBIQDAxsYGb7wxGF9+Wf1p7MbCaOdgLl26BI1Gg7Zt\n2xprl481nU6HNm2eRN++gbCzs8WVKyXIysrFnDlLcPt2manjUS38/X2xeXOS/vqMGVMxY8ZUfPvt\nOoSFTQAADBr0GtRqWyQkJJoqpskphBBC9k4qKysxbtw4PPXUU5g9e3a18YKCAvTu3Ruurn6wtLSW\nHYdM4PDhbaaOQJJYWFjA2dkBO3furHZ0Iv11MDqdDu+//z5UKhU++OAD2bsjoseI1EMkIQSioqJw\n6dIlxMbGQqVSydwdET1mpBbMzJkzkZ+fjzVr1qBJkyYyd0VEjyFpBVNYWIj169fDysoKgYGB+uUf\nfvghBg4cKGu3RPQYkVYwbm5uOHHihKzNE1EDwDc7EpE0LBgikoYFQ0TSsGCISBoWDBFJw4IhImlY\nMEQkDQuGiKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVD\nRNKwYIhIGhYMEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4IhImlYMEQk\nDQuGiKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVDRNKw\nYIhIGhYMEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBpLUwcAAK1WCwCoqio3cRKS\nxcKC/5eZKwsLBYD/+z2+12NRMMXFxf/7Z66Jk5Aszs4Opo5AkhUXF8PDw8NgmUIIIUyUR6+srAzZ\n2dlwdnaGUqk0dRwiegharRbFxcXw9vZGkyZNDMYei4IhIvPEA2MikoYFQ0TSsGCISBoWDBFJw4Ix\nsvj4eAwaNAje3t6YNm2aqeNQPaqoqMCMGTPQq1cv+Pv7IyQkBKmpqaaOZVKPxetgGpMWLVogLCwM\ne/fuRXk5X1hoTqqqquDq6oq1a9eiVatWSE1NxcSJE5GcnAx3d3dTxzMJFoyRvfTSSwCAX375BUVF\nRSZOQ/XJ1tYWkZGR+uu9evWCu7s7fv3110ZbMDxEIpLk0qVL0Gg0aNu2ramjmAwLhkiCyspKvPfe\newgNDYWnp6ep45gMC4aonul0Orz//vtQqVT44IMPTB3HpHgOhqgeCSEQFRWFS5cuITY2FiqVytSR\nTIoFY2RVVVXQarXQ6XTQarUoLy+HUqmEpSX/KszBzJkzkZ+fjzVr1lR7419jxDc7GtnixYuxZMkS\ng2UREREGzz5Qw1RYWIigoCBYWVkZ/Ifx4YcfYuDAgSZMZjosGCKShid5iUgaFgwRScOCISJpWDBE\nJA0LhoikYcEQkTQsGDOlVCrh5+cHb29vvPHGG7h169af3tbu3bvRv39/AMCmTZsQHR1d47olJSVY\nunTpQ+9j1qxZ+PTTT+u8/F6jR4/Gd999V+d9aTQaeHt7P3RGengsGDNlY2ODrKwsZGdnw8rKCsuW\nLTMYF0JAp9M99HYHDhz4wA/K+rMFQ+aJBdMIvPDCC8jLy4NGo0HHjh0RFhaGrl274uzZs0hJSUFA\nQAC6du2KN954A6WlpQCAbdu2oUOHDggMDMTGjRv124qLi0NERAQAoKioCKGhofD19YWvry/279+P\nadOmIT8/H35+fpgyZQoAYMGCBXjmmWfg4+ODmTNn6rc1b948eHl5oU+fPjhx4kSt9yM2NhbPPPMM\nfH19MXjwYINZ2Y4dO/DCCy+gffv22Lx5M4A739czZcoU/b6XL1/+6A8mPRQWjJmrqqrC1q1b0aVL\nFwDAiRMnMHLkSGRmZkKtVmPu3LnYsWMHjhw5gqeffhoLFy5EWVkZxo0bh+TkZOzduxcXLly477Yn\nTJiAnj174ujRozhy5Ag6d+6M6OhoeHp6IisrCwsWLEBKSgpOnjyJQ4cOISsrCxkZGdizZw8yMjKw\nbt06ZGZmYuPGjUhPT6/1vgwaNAjp6ek4evQoOnbsiFWrVunHNBoNUlNTsWXLFowfPx5lZWVYtWoV\nHBwckJ6ejvT0dMTGxuL06dP188BSnfAddmbq9u3b8PPzA3BnBjN27FicO3cOHh4e6NGjBwDg559/\nRk5ODp5//nkAdz5TNiAgAMePH0fr1q3Rrl07AMDbb7+NFStWVNvHTz/9hG+++QbAnXM+Dg4OuHr1\nqsE6KSkpSElJgb+/PwCgtLQUJ0+exI0bNxAaGgpbW1sAqNN7dbKzs/Hvf/8bJSUlKC0tRXBwsH5s\nyJAhsLCwQLt27dCmTRscP34cKSkpOHbsmP78zLVr13Dy5Em0b9++7g8kPRIWjJm6ew7mj9Rqtf5n\nIQT69u2LxMREg3WysrKgUCjqJYcQAtOnT8c//vEPg+Wff/75Q+9j9OjRSEpKgq+vL+Li4rB79279\n2B+3pVAoIITA4sWLDYoIuDPbIePgIVIj1qNHD+zbtw95eXkAgFu3buG3335Dhw4dcPr0aeTn5wNA\ntQK6q3fv3oiJiQFw53zH9evXYW9vjxs3bujXCQ4OxurVq/XndgoLC3Hx4kX89a9/xQ8//IDbt2/j\nxo0bSE5OrjXvjRs34OrqisrKSiQkJBiMbdiwATqdDvn5+Th16hS8vLwQHByMmJgYVFZWAgB+++03\n3Lx58yEfJXoUnME0Ys7OzoiLi8OwYcP033Awd+5ctG/fHitWrMCrr74KJycnBAYGIjs7u9rtv/ji\nC/z973/HqlWroFQqERMTg4CAADz//PPw9vbGK6+8ggULFiA3NxcBAQEAADs7O8THx6Nr164YOnQo\n/Pz84OHhgRdeeKHWvHPmzEH37t3h4eGBLl26GBSZl5cXevbsiaKiIixbtgxNmjTBO++8A41Gg65d\nu0IIAWdnZyQlJdXTo0d1wY9rICJpeIhERNKwYIhIGhYMEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgi\nkuZ/AI3Cl9A+CHL1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x122843a50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipeline = Pipeline([\n",
    "        ('ss', StandardScaler()),\n",
    "        ('pca', PCA(random_state=42)),\n",
    "        ('clf', lr)\n",
    "    ])\n",
    "grid_search = GridSearchCV(pipeline, lr_parameters, verbose=1, cv=5)\n",
    "grid_search.fit(data_train, label_train)\n",
    "print \"Logistic Regression Gridsearch\"\n",
    "print \"Grid Search Best Score:\", grid_search.best_score_\n",
    "print \"Grid Search Best Params:\", grid_search.best_params_\n",
    "print \"Grid Search Best Estimator:\", grid_search.best_estimator_\n",
    "    \n",
    "predictions = grid_search.best_estimator_.predict(data_test)\n",
    "print \"Classification Report:\", classification_report(label_test, predictions)\n",
    "print 'Accuracy: {0:.3f}'.format(accuracy_score(label_test, predictions))  \n",
    "    \n",
    "cnf_matrix = confusion_matrix(label_test, predictions)\n",
    "plot_confusion_matrix(cnf_matrix, classes=grid_search.best_estimator_.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "dataprocessed = ss.fit_transform(dataprocessed)\n",
    "\n",
    "# TRAIN AND TEST SPLIT WITH TEST SIZE OF 20%\n",
    "data_train,data_test,label_train,label_test = train_test_split(dataprocessed,label,test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params_fixed = {\n",
    "    'objective': 'binary:logistic',\n",
    "    'silent': 1,\n",
    "}\n",
    "\n",
    "# The space to search\n",
    "space = {\n",
    "    'max_depth': (1, 5),\n",
    "    'learning_rate': (10**-4, 10**-1),\n",
    "    'n_estimators': (10, 200),\n",
    "    'min_child_weight': (1, 20),\n",
    "    'subsample': (0, 1),\n",
    "    'colsample_bytree': (0.3, 1)\n",
    "}\n",
    "\n",
    "reg = XGBClassifier(**params_fixed)\n",
    "\n",
    "def objective(params):\n",
    "    \"\"\" Wrap a cross validated inverted `accuracy` as objective func \"\"\"\n",
    "    reg.set_params(**{k: p for k, p in zip(space.keys(), params)})\n",
    "    return 1-np.mean(cross_val_score(\n",
    "        reg, data_train, label_train, cv=5, n_jobs=-1,\n",
    "        scoring='accuracy')\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best accuracy score = 0.742519385523\n",
      "Best parameters = {'colsample_bytree': 0.29999999999999999, 'learning_rate': 0.10000000000000001, 'min_child_weight': 1, 'n_estimators': 32, 'subsample': 1, 'max_depth': 5}\n"
     ]
    }
   ],
   "source": [
    "from skopt import gp_minimize\n",
    "res_gp = gp_minimize(objective, space.values(), n_calls=50, random_state=2)\n",
    "best_hyper_params = {k: v for k, v in zip(space.keys(), res_gp.x)}\n",
    "\n",
    "print \"Best accuracy score =\", 1-res_gp.fun\n",
    "print \"Best parameters =\", best_hyper_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = best_hyper_params.copy()\n",
    "params.update(params_fixed)\n",
    "\n",
    "clf = XGBClassifier(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(data_train, label_train)\n",
    "y_test_preds = clf.predict(data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report:              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.73      0.90      0.81       126\n",
      "          2       0.72      0.45      0.55        74\n",
      "\n",
      "avg / total       0.73      0.73      0.71       200\n",
      "\n",
      "Accuracy: 0.730\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAAEvCAYAAACe8d6ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHk9JREFUeJzt3X1YVGXiPvB7eJU3UWIQASVFIBUBzUwSUxGl8g00MzWV\nzZc1QLMy06jFSsvW1jJdUVGhFddMI1ZtJVZD1CwlBY0AFWIS0BAlFESQYZ7vH/6cX4QIKs8MDPfn\nurhiznPmnHsGuXvOOTODQgghQEQkgZG+AxCR4WLBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4JpQ5Yu\nXYpOnTpBoVAgLi7ugbenUqmgUCjw448/Pni4VmDo0KGIiIjQd4xWRcHXwehXcXEx3n//fezduxeF\nhYWwt7eHt7c35s2bh2eeeabZ9pOZmYk+ffogISEBfn5+sLW1hYWFxQNts7a2FiUlJbC3t4eJiUkz\nJdUdlUqFbt26IS0tDf379290/dLSUpiamsLGxkYH6QxD6/tXYUBUKhUGDRoEGxsbfPDBB/Dx8YFG\no8GBAwcwd+5cnD9/vtn2lZubCwAIDg6GQqFolm0aGxvD0dGxWbbVkt28eRNmZmaws7PTd5TWR5De\nPP3006Jz586ivLy83lhpaan2+19//VUEBwcLa2trYW1tLUJCQkRBQYF2PCoqSvTu3Vts375ddO/e\nXVhbW4tx48aJkpIS7TiAOl9CCDFjxgwxatSoOvu9va3bTp8+LQICAoSNjY2wtrYW3t7e4ttvvxVC\nCJGfny8AiLS0NO36qampYsCAAcLc3Fw4ODiIBQsWiOrqau34kCFDxEsvvSSWLFkiHnroIaFUKsVr\nr70mamtrG3yeYmNjhZWVlfjvf/8rPD09hYWFhRgzZowoKysTO3fuFD169BDt27cXL7zwgqisrNTe\nb9++fcLf31906NBBdOzYUYwcOVJkZWVpx//8nAwZMqTO87JixQrh7OwslEqlNnt4eLgQQoicnBxh\naWkpPvvsszr7MzU1Fd9//32Dj6Wt4TkYPSktLUVSUhIiIiJgbW1db7xjx44AACEEgoODUVxcjG+/\n/RYpKSm4cOECgoODIf5wdKtSqbBjxw589dVXSE5ORnp6OiIjIwEACxcuRExMDADg4sWLuHjxYpNz\nTpkyBZ07d8bx48eRnp6OpUuXol27dndct6ioCE8//TT69u2L9PR0bN68Gdu3b8eSJUvqrLdt2zaY\nmJjg6NGjWLt2LT755BPs2LHjrjmqq6vxj3/8A9u2bcOBAwfw448/4tlnn8Vnn32GL7/8EomJidi7\ndy/WrVunvc/169exYMECHD9+HAcPHoStrS3GjBmDmzdvAgCOHz8OAEhKSsLFixeRkJCgvW9qaipO\nnz6NpKQkHDhwoF4eT09PfPzxx4iIiEBeXh5KSkoQGhqKyMhIDBw4sGlPblug74Zrq44dOyYAiISE\nhLuul5ycLIyMjER+fr52WV5enlAoFOJ///ufEOLWrMPc3FyUlZVp11m2bJlwc3PT3t65c6f484+7\nKTMYGxsbERcXd8dsf57BvPnmm8LNza3ObCQ2NlaYmZmJ69evCyFuzQIGDhxYZzuBgYFi5syZDT4H\nsbGxAoDIycnRLnvttdeEkZGRdpbW0OP5o4qKCmFkZCQOHz58x/x/3I69vb2oqqqqs/yPM5jbgoOD\nxeOPPy6eeeYZ8cQTTwi1Wt3g/tsizmD0RDTx3Hp2djacnJzw8MMPa5d1794dTk5OyMrK0i5zdXWF\nra2t9raTkxMuXbr0wDlfffVVzJo1CwEBAVi+fDlycnLumtXPzw9GRv//n5W/vz9u3rypPQcEAN7e\n3nXu15Ss5ubm8PT01N7u1KkTHB0dYW9vX2fZH7eTl5eHKVOmwM3NDe3bt0enTp2g0WiadG7Ly8sL\n5ubmja63adMm5Obm4tChQ4iPj4exsXGj92lLWDB64u7uDoVCgezs7LuuJ4Ro8KTsH5ebmprWG9No\nNHfdtpGRUb2iq6mpqXN76dKlyMrKQnBwMI4ePQpvb29s2bJF51n/fJVKoVA0up0xY8agpKQEGzZs\nwLFjx5Ceng4TExPtIdLdWFlZNboOcOvq3NWrV1FVVYWioqIm3actYcHoiZ2dHYKCgrB27VpUVFTU\nGy8rKwMA9OrVC0VFRVCpVNqxX375BRcuXECvXr0eKINSqax3PiYjI6Peeu7u7pg/fz6+/vprzJw5\nE5s2bbrj9nr16oXvv/++zi/5kSNHYGZmBjc3twfKeq+uXLmC7OxsvPnmmwgMDETPnj1RXl4OtVqt\nXcfMzAzArcvt96OsrAzTp0/HwoULER4ejmnTpuHatWvNkt9QsGD0aN26dRBCoH///ti5cyfOnDmD\nnJwcREdHaw8jAgMD4ePjg6lTp+LEiRP48ccfMXXqVPTr1w8BAQEPtP+AgACkp6djy5YtyM3Nxd//\n/nd899132vEbN24gPDwcBw8ehEqlwrFjx3DkyJEGiy0sLAwXLlxAWFgYsrOz8fXXX2Px4sWIiIiA\npaXlA2W9Vx07doS9vT1iYmKQm5uL1NRUzJ07t85MyMHBARYWFvjmm29QXFyMq1ev3tM+5s6dC3t7\ne7z77rv48MMPYWNjg/Dw8OZ+KK0aC0aPunXrhpMnT2LEiBF444034O3tjYCAAOzevRsbNmwAcGva\nn5iYCKVSiaFDh2LYsGFwdHREYmLiA7+eJSgoCFFRUYiMjMSjjz4KlUqFsLAw7bixsTF+//13zJgx\nA56enggJCYGfnx9WrVp1x+05Oztj3759SE9Ph6+vL1588UVMnjwZ77///gPlvB9GRkbYsWMHTp8+\nDS8vL4SHh+O9996rc17FxMQEn376KTZt2gQnJyeMGzeuydvfunUrdu/ejW3btsHU1BTm5ub497//\njV27dmH79u0yHlKrxFfyEpE0nMEQkTQsGCKShgVDRNKwYIhImhbxbuqqqipkZmZCqVTylZBErczt\nj+3w8vKq9z61FlEwmZmZmDp1qr5jENED2LZtW73P1WkRBaNUKgHcCtgWPl+kLerWrZu+I5AkJiYm\n6Nq1q/b3uM6YHvLUc/uwyNHRES4uLnpOQzL88SX6ZJjudHqDJ3mJSBoWDBFJw4IhImlYMEQkDQuG\niKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVDRNKwYIhI\nGhYMEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4IhImlYMEQkDQuGiKRh\nwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVDRNKwYIhIGhYM\nEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgikoYF00wOHTqEsWPHwtnZGQqFAnFxcXXGExISEBQUBKVS\nCYVCgYMHD9bbxuzZs+Hm5gYLCwsolUqMGzcO2dnZunkAdM8GDx6M//znPygsLIQQAjNmzKgz/u67\n7yI7OxsVFRUoLS3F/v374efnp6e0+sGCaSYVFRXw8vLC6tWrYWFhUW/8+vXreOKJJ7Bq1aoGt9G/\nf3/ExcUhOzsb33zzDYQQCAwMRE1NjczodJ+sra2RmZmJl19+GZWVlfXGz5w5g/DwcPTp0wf+/v7I\nz89HUlISHBwc9JBWT0QLUFBQIDw8PERBQYG+ozQLKysrERsbe8exkpISAUCkpKQ0up1Tp04JACIn\nJ6d5A+oBAIP+Ki8vFzNmzLjrOjY2NkIIIUaOHKn3vM35ZWJi0uDvL2cwLdT169cRGxuLrl274uGH\nH9Z3HHpApqammDNnDq5evYqMjAx9x9EZFkwLs27dOlhbW8Pa2hr79u3DgQMHYG5uru9YdJ9GjRqF\n8vJyVFVV4ZVXXsGIESNw6dIlfcfSGRZMCzN16lSkp6cjNTUVHh4emDhx4h2P76l1SElJga+vL554\n4gkkJSXhiy++gKOjo75j6YzUgomPj8f48ePh5eWFxYsXy9yVwbC1tYW7uzuefPJJ7Nq1C2fPnsWX\nX36p71h0nyorK5GXl4djx45h1qxZqKmpwaxZs/QdS2dMZG7cwcEBYWFhOHz4MKqrq2XuyiAJISCE\n4HNnQIyMjNrUIa/Ughk5ciQA4KeffkJxcbHMXeldRUUFcnNzAQAajQbnz59HRkYG7Ozs0LVrV5SW\nluL8+fMoKysDAOTm5qJDhw5wdHSEo6MjcnNz8eWXXyIwMBBKpRKFhYVYsWIFzM3NMXr0aH0+NGqA\nlZUVevToAeBWcXTt2hU+Pj4oLS1FWVkZFi1ahD179uDixYtQKpUIDw+Hi4sLvvjiCz0n1yFdXKJc\ntWqVeOONNxocN4TL1CkpKXe8hDdjxgwhhBCxsbF3HI+KihJCCHH+/Hnx1FNPCaVSKUxNTYWLi4uY\nMmWKyM7O1t+DakZ3euyt/WvIkCF3fKyxsbHCwsJCJCQkiKKiIlFVVSWKiopEYmKiGDBggN5zN/fX\n3S5TS53BtCVDhw7Frd+jOwsNDUVoaGiD4126dMG+ffskJCNZUlNToVAoGhwfP368DtO0TLyKRETS\nsGCISBqph0hqtRq1tbXQaDSora1FdXU1jI2NYWLCIzOitkDqDCY6Ohre3t7YuHEjdu/eDW9vb0RH\nR8vcJRG1IFKnEvPmzcO8efNk7oKIWjCegyEiaVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoW\nDBFJw4IhImlYMEQkDQuGiKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQ\nkTQsGCKShgVDRNKwYIhIGhYMEUnDgiEiaVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSmDQ0cO3atbve\nsX379s0ehogMS4MF07t3bygUCgghtMtu31YoFDh//rxOAhJR69VgwRQUFOgyBxEZoCadg/n888/x\n/vvvAwAKCwtx4sQJqaGIyDA0WjARERFISUnB1q1bAQCWlpaYO3eu9GBE1Po1eIh029GjR3Hy5En0\n7dsXAGBnZ4ebN29KD0ZErV+jMxhTU1NoNBooFAoAwJUrV2BkxKvbRNS4RpsiPDwcEyZMQElJCaKi\nouDv74833nhDF9mIqJVr9BBp+vTpePTRR7F//34AwM6dO+Hl5SU9GBG1fo0WDADU1tbC1NQUCoUC\nGo1GdiYiMhCNHiItX74ckydPxoULF1BYWIgpU6bggw8+0EU2ImrlGp3BxMfH48SJE7C0tAQAREZG\n4tFHH8WSJUukhyOi1q3RGYyrqyvUarX2tlqtRvfu3aWGIiLD0OAM5pVXXoFCoYClpSV69+6NoKAg\nKBQKJCcnw9/fX5cZiaiVarBgbl8p6t27N0aNGqVdPnDgQPmpiMggNFgwM2fO1GUOIjJAjZ7kzcvL\nQ2RkJLKyslBVVaVdfvbsWanBiKj1a/Qkb2hoKP7yl79ACIF9+/bhueeew/PPP6+LbETUyjVaMJWV\nlQgKCgIAuLm5YdmyZUhJSZEejIhav0YPkczNzSGEgJubG9avXw9nZ2dcunRJF9mIqJVrtGA+/vhj\nVFRU4NNPP0VkZCSuXr2KLVu26CIbEbVyjRbM448/DgCwsbHRfugUEVFTNFgwISEh2s+AuZOEhAQp\ngYjIcDRYMBEREbrMAQCYNCkCJibmOt8vyefl9aS+I5AkQqhRU3P5jmMNFszw4cOlBSKitoGffUlE\n0rBgiEiaJhdMdXW1zBxEZIAaLZjjx4+jT58+cHd3BwCcOnUK8+bNkx6MiFq/Rgtm/vz52Lt3Lx56\n6CEAgI+PD98qQERN0mjBaDQauLq61llmbGwsLRARGY5GX8nbpUsXHD9+HAqFArW1tVizZg08PDx0\nkY2IWrlGZzDR0dFYtWoVzp8/j06dOuGHH35AdHS0LrIRUSvX6AzGwcEBn3/+uS6yEJGBabRgZs+e\nfcf3JG3cuFFKICIyHI0WTGBgoPb7qqoqfPXVV+jSpYvUUERkGBotmEmTJtW5PW3aNIwYMUJaICIy\nHPf8VoH8/Hz8+uuvMrIQkYFpdAbTsWNH7TkYjUYDOzs7rFixQnowImr97lowQgicOnUKzs7OAAAj\nI6O7fggVEdEf3fUQSaFQICQkBMbGxjA2Nma5ENE9afQczIABA3Dy5EldZCEiA9PgIZJarYaJiQmO\nHDmCmJgYuLm5wcrKCkIIKBQKlg4RNarBgrk9c0lMTNRlHiIyIA0WjBACwK2/5khEdD8aLJiSkhKs\nWrWqwTu++uqrUgIRkeFosGBqa2tRUVGhnckQEd2rBgumc+fO+Nvf/qbLLERkYBq8TM2ZCxE9qAYL\n5sCBA7rMQUQGqMGCsbOz02UOIjJA/MNrRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4Ih\nImlYMEQkDQuGiKRhwRCRNCwYIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKS\nhgVDRNKwYIhIGhYMEUnT4J+Opeb1wgvBmD17MhISkrB6dSwAYPDgARg7NhAeHt3QoUN7vPzyO8jI\nyNJzUmqK558PxsSJY+Hk5AgAyMtTYcOGf+Hw4R8AABERL2LkyKHo1MkBarUaWVlnsXbtZpw69bM+\nY+scZzA60KuXO0aPHo7c3F/rLLewMEdm5ln885//0lMyul/FxSX4+OMNeO652Xj++Tk4duwkVq9e\nDg+P7gAAlaoAy5d/ggkT/oLp0yNQVHQR69evxEMPddRzct3iDEYyKysLvPVWBD78cD1mzJhQZyw5\n+TAAwNbWRh/R6AGkpHxX5/aaNZswadI4+Pj0xtmzv2Dv3v/VGV+58p+YMGE0PD174OjRNF1G1SvO\nYCRbuHAOUlOPIT29bU2N2xIjIyM89VQALC0tkJFR/+dsYmKCZ58dg/LyCpw5k6uHhPrDGYxEo0cH\nwNnZEcuXr9V3FJLA3b074uP/CTMzM1RW3sDLL7+Fc+d+0Y4/+aQfVq78G9q1a4eSkiuYM+c1XLny\nux4T6x5nMJJ06dIZs2dPxrJla6BW1+o7DkmQn38ezz47C1OnhuGLL/6D5cuXoEePbtrxtLR0PPvs\nLEybFo7vvjuOjz5aCnt7Oz0m1j1pM5ibN29i6dKl+P7771FWVgZXV1e88sorGDJkiKxdtii9e3ug\nQ4f2iI39SLvMxMQYPj49MXbsCDz11HTU1Kj1mJAelFqtRkFBEQAgK+sMvLwewbRpExEV9XcAwI0b\nVSgoKEJBQRFOn87C3r3bMGHCaGzY0HZO6ksrGLVajc6dO2Pr1q1wcnJCamoqFixYgD179sDFxUXW\nbluMI0fSEBq6sM6yxYtfQmHhRcTHJ7JcDJBCYQQzM9MGx42MFHcdN0TSCsbS0hLz5s3T3h42bBhc\nXFzw888/t4mCqaioREVFZZ1lN25U4dq1CuTnFwAAbGys0KmTPaytrQAAzs6OqKi4jtLSMpSWXtV5\nZmq6BQvm4NChH/Dbb5dgZWWJZ54Zjsce80V4+GJYWVnixRcn4+DBoygpuQI7uw54/vkQdOqkxDff\npOg7uk7p7CTv5cuXoVKp0KNHD13tssUbNKg/liwJ095etOivAIDY2J2Ii9ulr1jUBPb2dvjgg0jY\n29uhvPw6zp3Lw0svLcLRo2lo184cbm7dEBz8DDp0aI+ysmv4+ecchIbOx9mzvzS+cQOiEEII2Tup\nqanB7Nmz0bVrV7z77rv1xgsLCzF8+HDY2/eEiYm57DikB6Wlv+k7AkkihBo1NZdx4MCBekcn0q8i\naTQaLFq0CKampnj77bdl746IWhCph0hCCERGRuLy5cuIiYmBqWnbOsFF1NZJLZioqCjk5eUhNjYW\n7dq1k7krImqBpBVMUVERduzYATMzM/j7+2uXv/POOxg7dqys3RJRCyKtYJydnXHmzBlZmyeiVoBv\nFSAiaVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4IhImlYMEQkDQuGiKRhwRCRNCwY\nIpKGBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVDRNKwYIhIGhYMEUnDgiEi\naVgwRCQNC4aIpGHBEJE0LBgikoYFQ0TSsGCISBoWDBFJw4IhImlYMEQkDQuGiKRhwRCRNCwYIpKG\nBUNE0rBgiEgaFgwRScOCISJpWDBEJA0LhoikYcEQkTQsGCKShgVDRNKwYIhIGhYMEUnDgiEiaVgw\nRCQNC4aIpGHBEJE0JvoOAAC1tbX/77839ZyEZBFCre8IJIkQt39/a+uNtYiCKSkpAQD8/nuenpMQ\n0f0qKSmBq6trnWUKIYTQUx6tqqoqZGZmQqlUwtjYWN9xiOge1NbWoqSkBF5eXmjXrl2dsRZRMERk\nmHiSl4ikYcEQkTQsGCKShgVDRNKwYHQsPj4e48ePh5eXFxYvXqzvONSMbt68iTfffBPDhg1D3759\nERwcjNTUVH3H0qsW8TqYtsTBwQFhYWE4fPgwqqur9R2HmpFarUbnzp2xdetWODk5ITU1FQsWLMCe\nPXvg4uKi73h6wYLRsZEjRwIAfvrpJxQXF+s5DTUnS0tLzJs3T3t72LBhcHFxwc8//9xmC4aHSESS\nXL58GSqVCj169NB3FL1hwRBJUFNTg4ULFyIkJARubm76jqM3LBiiZqbRaLBo0SKYmpri7bff1ncc\nveI5GKJmJIRAZGQkLl++jJiYGJiamuo7kl6xYHRMrVajtrYWGo0GtbW1qK6uhrGxMUxM+KMwBFFR\nUcjLy0NsbGy9N/61RXyzo46tWbMGa9eurbMsIiKiztUHap2KiooQEBAAMzOzOv/DeOeddzB27Fg9\nJtMfFgwRScOTvEQkDQuGiKRhwRCRNCwYIpKGBUNE0rBgiEgaFoyBMjY2hq+vL7y8vDBx4kRUVlbe\n97YOHjyI0aNHAwB2796NFStWNLhuWVkZ1q1bd8/7WLp0KT766KMmL/+j0NBQ7Nq1q8n7UqlU8PLy\nuueMdO9YMAbKwsICGRkZyMzMhJmZGdavX19nXAgBjUZzz9sdO3bsXT8o634LhgwTC6YNGDx4MHJz\nc6FSqdCzZ0+EhYWhX79+KCgoQHJyMvz8/NCvXz9MnDgRFRUVAICkpCQ88sgj8Pf3R0JCgnZbcXFx\niIiIAAAUFxcjJCQEPj4+8PHxwdGjR7F48WLk5eXB19cXr7/+OgBg5cqVeOyxx+Dt7Y2oqCjttpYv\nXw5PT08EBgbizJkzjT6OmJgYPPbYY/Dx8cGECRPqzMr279+PwYMHw8PDA3v37gVw6+/1vP7669p9\nb9iw4cGfTLonLBgDp1arsW/fPvTp0wcAcObMGUyfPh3p6emwsrLCsmXLsH//fpw8eRL9+/fHqlWr\nUFVVhdmzZ2PPnj04fPgwfvvttztue/78+RgyZAhOnTqFkydPonfv3lixYgXc3NyQkZGBlStXIjk5\nGefOncPx48eRkZGBEydO4NChQzhx4gQ+//xzpKenIyEhAWlpaY0+lvHjxyMtLQ2nTp1Cz549sXnz\nZu2YSqVCamoqvv76a8ydOxdVVVXYvHkzbG1tkZaWhrS0NMTExCA/P795nlhqEr7DzkDduHEDvr6+\nAG7NYGbOnIkLFy7A1dUVAwcOBAD88MMPyMrKwqBBgwDc+kxZPz8/5OTkoFu3bnB3dwcAvPDCC9i4\ncWO9fXz77bf417/+BeDWOR9bW1v8/vvvddZJTk5GcnIy+vbtCwCoqKjAuXPnUF5ejpCQEFhaWgJA\nk96rk5mZibfeegtlZWWoqKhAUFCQduy5556DkZER3N3d0b17d+Tk5CA5ORmnT5/Wnp+5evUqzp07\nBw8Pj6Y/kfRAWDAG6vY5mD+zsrLSfi+EwIgRI7B9+/Y662RkZEChUDRLDiEElixZgr/+9a91ln/y\nySf3vI/Q0FAkJibCx8cHcXFxOHjwoHbsz9tSKBQQQmDNmjV1igi4Ndsh3eAhUhs2cOBAfPfdd8jN\nzQUAVFZW4uzZs3jkkUeQn5+PvLw8AKhXQLcNHz4c0dHRAG6d77h27RpsbGxQXl6uXScoKAhbtmzR\nntspKirCpUuX8OSTT+Krr77CjRs3UF5ejj179jSat7y8HJ07d0ZNTQ22bdtWZ2znzp3QaDTIy8vD\nL7/8Ak9PTwQFBSE6Oho1NTUAgLNnz+L69ev3+CzRg+AMpg1TKpWIi4vD5MmTtX/hYNmyZfDw8MDG\njRsxatQo2Nvbw9/fH5mZmfXuv3r1asyZMwebN2+GsbExoqOj4efnh0GDBsHLywtPP/00Vq5ciezs\nbPj5+QEArK2tER8fj379+mHSpEnw9fWFq6srBg8e3Gje9957D48//jhcXV3Rp0+fOkXm6emJIUOG\noLi4GOvXr0e7du0wa9YsqFQq9OvXD0IIKJVKJCYmNtOzR03Bj2sgIml4iERE0rBgiEgaFgwRScOC\nISJpWDBEJA0LhoikYcEQkTQsGCKS5v8AaJu2GYR3JbwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11f965710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print \"Classification Report:\", classification_report(label_test, y_test_preds)\n",
    "print 'Accuracy: {0:.3f}'.format(accuracy_score(label_test, y_test_preds))  \n",
    "cnf_matrix = confusion_matrix(label_test, y_test_preds)\n",
    "plot_confusion_matrix(cnf_matrix, classes=grid_search.best_estimator_.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "PCA for dimensionality reduction\n",
      "\n",
      "\n",
      "('The features selected in order of importance are - ', array([[ 27.]]))\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfAAAAEhCAYAAACAzvJsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1cjffjP/BXpehU38yEfZaG6pxQKveJJkQ2dxkfvsjc\njBFKNHMz9z6zj1GU8DEbw2z4TBFzt2ZKi9zMB1tnqumGfcxNyjqHUr1/f/TrmmsVJb7Hdbyej8ce\n23nfXe/rXO28zvs617mOiRBCgIiIiBTF1NATICIioppjgBMRESkQA5yIiEiBGOBEREQKxAAnIiJS\nIAY4ERGRAjHA6bmUlZUFjUaD8+fPG3oqT81vv/2GwMBAuLu7o1WrVgadi4+PDzZu3GjQORBR7TDA\nqcYmT56MIUOGVFpXWFiIjh07IiIiolbbsLe3x4kTJ9C6detajfM8WbduHe7evYu9e/ciISHB0NOh\nZywwMBDz5s0z9DTIiDHAqcaGDRuGixcvQqvVVqg7fPgw/vjjDwwdOvSJxy8qKoKZmRns7Oxgbm5e\nm6k+V7KysuDu7o5mzZqhYcOGhp4OESkcA5xqzMfHB3/729+wa9euCnW7d++Gt7c37O3tAQB79+7F\nkCFD0K5dO3Tq1AnvvvsusrKypPblp8r379+P8ePHw93dHdHR0ZWeQl+5ciX8/f3h7u6O7t27Y9Gi\nRSgoKJBt283NDWfOnMHAgQPh7u6Ot956C5cuXZLNMTMzE9OmTUPHjh3h7u6OAQMG4Pjx41L9hQsX\nMHbsWHh6esLLywvBwcH473//+8jn5I8//sAHH3yAzp07w83NDUOGDEFycjIAoLi4GBqNBikpKdi5\ncyc0Gs0jV2aP2352djamTJmCrl27wt3dHf3790dcXJxsDCEEtm7dir59+8LV1RVdunTB9OnTZW2K\nioqwdOlSdOjQAV26dME///lPlJSUPHI/b926hdmzZ8PLywtubm7w9/dHTEyMVH/u3DmMGDECbdq0\nQceOHREWFobc3FypPiIiAv7+/ti/fz969eoFd3d3TJs2DTqdDgcPHkSfPn3Qtm1bhISEyI5tWFgY\nxo8fj02bNqFr167w8PBASEgI7t69K9vnjRs3okePHnB1dUWvXr2wbds22fx9fHywdu3aR+63EAKf\nf/45+vTpAzc3N/Tp0wf/+te/UFxcXO1xwsLCkJKSgn//+9/QaDTQaDQ4c+bMI59bohoTRE8gKipK\ntGvXTty7d08qu3LlilCr1eLIkSNS2a5du8SxY8dEVlaWuHTpkpgwYYLo06ePKCoqEkIIkZmZKdRq\ntfDx8RH79u0T2dnZIicnRyr/8ccfpbHWrl0rTp8+LXJyckRSUpLo3bu3mDNnjmxbLi4uYuTIkeLM\nmTMiPT1djB07VvTq1UsUFxcLIYS4fv266Ny5sxg7dqw4e/asyMrKEkePHhUJCQlCCCG0Wq1wd3cX\nUVFRIj09XaSmpoqpU6eKPn36iMLCwiqfj6CgINGjRw+RmJgo0tPTxeLFi0Xr1q3FlStXhBBC3Lhx\nQwwZMkTMmjVL3LhxQ/zxxx+VjlOd7aemport27eL1NRUkZWVJTZv3ixcXFxESkqKNE54eLjw8PAQ\n27dvF7/++qu4ePGiWL9+vVTfrVs30b59e/HJJ5+IK1euiP3794uWLVuKPXv2VLmPOp1O9O7dWwQE\nBIgffvhBZGdni4SEBHHgwAHpufXw8BBhYWHil19+ESkpKeKNN94Qo0aNqjCvd999V2i1WnHy5EnR\nsWNHMW7cODFx4kSRmpoqUlJSRKdOnUR4eLjUb+bMmcLT01MEBQUJrVYrkpOTRc+ePcW0adOkNlu2\nbBFt2rQRu3btEleuXBHbt28XrVu3lu1TdfY7PDxc+Pr6iqNHj4rs7Gxx7Ngx0a1bNxEVFVXtce7e\nvSuGDRsmZsyYIW7cuCFu3Lgh/c0TPS0McHoi169fFy1bthQxMTFS2YoVK4S3t7d48OBBlf1u374t\n1Gq1OH/+vBDizwDfsGGDrF1lAf5X33zzjWjTpo0oLS0VQpQFuFqtFqmpqVKbM2fOCLVaLbKysoQQ\nQqxcuVJ4e3sLvV5f6ZgzZ84UM2fOlJXdu3dPtG7dWnz33XeV9snIyBBqtVokJiZKZaWlpaJ///7i\ngw8+kMqGDx8u5s+fX+X+POn2hRBiwoQJYsGCBUIIIf744w/h6uoqNm/eXGX7bt26iSlTpsjKxowZ\nI8LCwqrs8+WXX4o2bdqI33//vdL6lStXiu7du8uC6uLFi0KtVouzZ88KIcrCsXXr1uLOnTtSm/nz\n54uWLVuK3NxcqWzRokVi6NCh0uPyAH/4jc/3338vNBqNyMnJEUII4e3tLVatWiWb05IlS0Tv3r2r\nvd8FBQXCzc1NJCUlydrs3r1bdOzYsdrjCCHEqFGjxNy5cyt9roiehjqGPgNAytS4cWO8/vrr2LVr\nFwYNGoQHDx4gNjYWQ4YMQZ06f/5Z/fTTT4iOjoZWq8WdO3ek8mvXrsHd3V163KZNm8du89ChQ9i6\ndSuys7Oh0+lQUlKCwsJC5Obm4uWXXwYA1KlTB2q1WurTqFEjAGWnfh0cHPDTTz+hXbt2sLS0rHQb\nFy9exLVr1xAfHy8rLy4ulp36f1h6ejoAoH379lKZiYkJ2rdvj59//vmx+1XT7ev1ekRHR+P777/H\nzZs38eDBAxQVFaFLly4AgMuXL6OoqAhdu3Z95LZcXFxkjxs1aoSbN29W2f6nn36Cs7Oz9Jz+VVpa\nGjw8PGTXLbi6ukKlUiE9PR1t27YFADRp0gT169eX2jRs2BCNGzfGSy+9JCu7ffu2bHy1Wg1ra2vp\ncdu2bSGEQEZGBmxsbHDz5k3ZMQCADh06YMeOHSgqKoKFhcVj9/vy5csoLCxEUFAQTExMpDblf2v5\n+fmwtbV97DhE/xcY4PTEhg0bhnfffRcZGRlIT09Hbm6u7OK1goICjBs3Dp06dcLy5cvRsGFDlJaW\nol+/fnjw4IFsrKoCtdzZs2cRGhqKSZMmYfbs2bCxscHZs2cxb9482VimpqYwNf3z0o7yF+HS0tIK\nZZURQmDw4MEYP358hbqHA+avKhtTCPHIbT3p9j/66CMkJCTg/fffR/PmzWFpaYkPP/wQhYWFj53T\nw8oD7eH2Dz9PlXncmFXVP1z+8Bu88rrKykQNfiixvO1ft1/ZGI/a7/J/r127Fk2bNq3Q9+E3EE/y\n/BE9TQxwemIPX8yWkZEhu3gNADIyMpCXl4cZM2agWbNmAIDTp08/0bbOnj2Lhg0bIiQkRCo7cOBA\njcdp3bo1YmNjce/evUrfNLi6uuKXX36Bg4NDtcPXyckJQgicOXNGWvUKIXD27Fl4eHjUaH7V2f7p\n06cxcOBA9O3bF0DZ6jAzMxOvvPIKgLKVqrm5ORITE+Ho6Fij7T9K69atsW/fPty4caPSVbizszP2\n79+PBw8eSKvwS5cuQa/Xw8nJqdbbT0tLg06ng5WVFYCyC+YAoEWLFqhfvz7s7Oxw+vRpdOvWTepz\n5swZODg4VAjbqmg0GlhYWCAnJ+exZzAex9zcnIFOzxSvQqcnZmpqirfeegtff/01kpKSMGzYMFn9\nq6++CnNzc2zbtg05OTlISkrC8uXLn2hbLVq0wK1bt7Bnzx7k5OTg66+/xs6dO2s8zqhRo1BUVISp\nU6fi3LlzyMnJwXfffYfExEQAwKRJk3D58mW8//77uHDhAnJycpCcnIylS5fi2rVrVc7Nz88PCxcu\nRFJSEjIyMrB06VL8+uuvGDduXI3mV53tN2/eHN9++y0uXLiAtLQ0fPDBB7h165Y0hrW1NcaMGYM1\na9Zgx44dyMzMRGpqaq1v3NK/f380atQIkyZNQnJyMnJycvDDDz/g4MGDAMqe27y8PMydOxdpaWk4\nc+YMZs+ejY4dO8LT07NW2wbK3hTNnj0bly9fxqlTp7Bs2TL07t1bWilPnDgRn3/+OXbv3o3MzEzs\n2LEDO3fuxKRJk6q9DWtra0yYMAErV67Ejh07cOXKFaSlpSEuLg6rVq2q0Xzt7e1x6dIlZGdnIzc3\nV3YVO9HTwACnWhk6dCj0ej1efvll+Pr6yuoaNmyIFStWICEhAW+88QZWrlyJuXPnPtF2evXqhXfe\neQcrV65E//79ceTIEbz33ns1Hqdx48bYsWMHLCwsMGHCBPTr1w9r1qyRTrWq1Wp8+eWXuHv3LsaN\nG4c333wTCxYswIMHD2BjY1PluMuXL4eXlxdmzJiBgQMH4sKFC/jkk0+kMw/VVZ3tz5s3D40aNcLo\n0aMxduxYvPrqq/Dz85ONM3PmTAQHB+Pzzz9Hv379MH78+Eq/t18TVlZW2L59O1q0aIHp06fjjTfe\nwJIlS6RT940bN8ann36Kq1evYvDgwZg8eTJcXFywZs2aWm23nKenJ9zc3DB27FhMnDgRLVu2xNKl\nS6X6wMBATJkyBevXr8ebb76Jzz77DLNmzUJAQECNthMcHIxZs2bhq6++Qv/+/TFixAhs27YNr776\nao3GGT9+PKytrTFw4EB4eXkZ1V0F6flgImryQRMRkQGEhYXhzp07+PTTTw09FaLnBlfgRERECsQA\nJyIiUiCeQiciIlIgrsCJiIgUiN8DB3D//n1cunQJdnZ2MDMzM/R0iIheaCUlJbh58yZcXV1Rr149\nQ0/nucUAR9nNJkaOHGnoaRAR0UO++OKLCrfHpT8xwAHY2dkBKPtjadKkiYFnQ0T0Yrt+/TpGjhwp\nvTZT5RjggHTavEmTJrJbgRIRkeHwI81Hq9ZFbFevXsWECRPQoUMHeHt7Y8mSJdJtAVNTUzF48GC4\nu7tj8ODBSE1NlfoJIfDxxx+jU6dO6NSpE1asWCH7cQFD9SUiIlK6agX44sWL8fLLL+PEiROIjY3F\n6dOnpZ/oCwoKwoABA3D69GkMGjQIQUFBKCoqAgDs3LkT3377Lfbu3Yt9+/bh+++/x1dffQUAButL\nRERkDKq9Au/bty/q1q0LOzs7dO3aFenp6UhJSUFxcTHefvttWFhYYPTo0RBC4OTJkwCA2NhYjBs3\nDk2aNEHjxo0xduxYxMTEAIDB+hIRERmDagX46NGjceDAAdy7dw+///47EhMT0a1bN6Snp0Oj0ch+\n9lCj0SA9PR1A2c//Pfyj9y4uLkhLSwMAg/UlIiIyBtUK8I4dOyI9PR3t2rWDj48PXF1d0atXL+h0\nugq/0GRtbQ2dTgcA0Ov1sLa2lupsbGyg1+shhDBYXyIiImPw2AAvLS3F+PHj4efnh/Pnz+PkyZPI\nz8/Hxx9/DCsrKxQUFMja63Q6WFlZAQBUKpUUqgBQUFAAlUoFExMTg/V9FpJOnMKM0LlIOnHqmYxP\nRET0V48N8Ly8PPz3v//FqFGjYGFhgZdeeglvvfUWEhIS4OTkhF9++UW2sv3ll1/g5OQEAHB2dpb9\nBrFWq4WzszMAGKzv09bX/y2MGvUOYmLiMGrUO+jr/9Yz2Q4REdHDHhvgDRo0gL29Pb788ksUFxfj\n7t27iImJgUajQceOHWFmZoatW7eiqKgI27dvBwB07twZADBw4EBs3rwZv//+O37//Xds3rwZAQEB\nAGCwvk/TiRPJ0Govy8q02stciRMR0TNXrc/A165di8TERHh5ecHPzw9mZmaYO3cuLCwsEB0djb17\n96J9+/b4+uuvER0dDQsLCwDA8OHD4evri/79+6N///54/fXXMXz4cAAwWN+nac/XcZWWf/313qe+\nLSIioofx50RR9jW5nj17Ij4+vkZ3Yks6cQqjRr1ToXz79k3w7trpaU6RiOiF8aSvyS8a/pxoLXh3\n7QQXF7WsrGVLNcObiIieOQZ4LR089DWGDBkAABgyZAC+Ofi1gWdEREQvAgb4U9DUoans30RERM8a\nA5yIiEiBGOBEREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkREpEAMcCIiIgVigBMRESkQA5yIiEiB\nGOBEREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkREpECPDXBPT0/ZPy1btsTSpUul+uTkZPj7+8Pd\n3R2BgYG4du2aVFdUVIQ5c+agbdu28Pb2xubNm2VjG6ovERGR0j02wH/88Ufpn6SkJNSrVw/+/v4A\ngNzcXEydOhUhISFISUmBq6srQkNDpb5RUVHIysrCsWPHsHXrVmzatAkJCQkG7UtERGQManQK/fDh\nw2jQoAHat28PADh69CicnZ3Rt29f1K1bF9OmTYNWq0VGRgYAIDY2FkFBQbC1tYWjoyOGDh2KmJgY\ng/YlIiIyBjUK8JiYGAwaNAgmJiYAgLS0NGg0GqlepVLBwcEB6enpyM/Px40bN+Di4iLVu7i4ID09\n3aB9iYiIjEG1A/y3337D6dOnMWjQIKlMr9fDxsZG1s7a2ho6nQ56vR4AZPU2NjbQ6XQG7UtERGQM\nqh3gsbGxaNeuHZo2bSqVqVQqFBQUyNrpdDpYWVlBpVIBgKy+oKAAVlZWBu1LRERkDKod4Hv37pWt\nvgHA2dkZWq1WeqzX65GdnQ0nJyfY2trCzs5OVq/VauHk5GTQvkRERMagWgF+7tw5/P7779LV5+X8\n/PyQlpaGw4cPo7CwENHR0dBoNHB0dAQADBo0COvXr0d+fj4yMjKwe/duBAQEGLQvERGRMahWgMfG\nxsLPzw/W1tay8gYNGiAqKgoRERHo0KEDLly4gPDwcKk+ODgYTZs2ha+vLwIDAzF+/Hj4+PgYtC8R\nEZExMBFCCENPwtCuXr2Knj17Ij4+Hvb29jXuHxm5ARHh0QidMQXBwZOewQyJiF4ctX1NflHwVqpE\nREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkREpEAMcCIiIgVigBMRESkQA5yIiEiBGOBEREQKxAAn\nIiJSIAY4ERGRAjHAiYiIFIgBTkREpEAMcCIiIgVigBMRESkQA5yIiEiBGOBEREQKxAAnIiJSoGoH\n+IEDB9C3b194eHigV69eOHPmDAAgOTkZ/v7+cHd3R2BgIK5duyb1KSoqwpw5c9C2bVt4e3tj8+bN\nsjEN1ZeIiEjpqhXgSUlJWLlyJZYvX45z587hiy++QNOmTZGbm4upU6ciJCQEKSkpcHV1RWhoqNQv\nKioKWVlZOHbsGLZu3YpNmzYhISEBAAzWl4iIyBhUK8CjoqIQFBQEDw8PmJqaonHjxmjcuDGOHj0K\nZ2dn9O3bF3Xr1sW0adOg1WqRkZEBAIiNjUVQUBBsbW3h6OiIoUOHIiYmBgAM1peIiMgYPDbAS0pK\ncOnSJdy5cwd+fn7w8fHBkiVLcP/+faSlpUGj0UhtVSoVHBwckJ6ejvz8fNy4cQMuLi5SvYuLC9LT\n0wHAYH2JiIiMwWMD/NatW3jw4AEOHTqEL774ArGxsfj555+xfv166PV62NjYyNpbW1tDp9NBr9cD\ngKzexsYGOp0OAAzWl4iIyBg8NsDr1asHAAgMDESjRo3QoEEDjB07FsePH4dKpUJBQYGsvU6ng5WV\nFVQqFQDI6gsKCmBlZQUAButLRERkDB4b4La2tmjSpAlMTEwq1Dk7O0Or1UqP9Xo9srOz4eTkBFtb\nW9jZ2cnqtVotnJycDNqXiIjIGFTrIrbBgwdj27ZtuH37NvLz8/H555+je/fu8PPzQ1paGg4fPozC\nwkJER0dDo9HA0dERADBo0CCsX78e+fn5yMjIwO7duxEQEAAAButLRERkDKoV4EFBQXBzc0OfPn3w\nxhtvoFWrVpg8eTIaNGiAqKgoREREoEOHDrhw4QLCw8OlfsHBwWjatCl8fX0RGBiI8ePHw8fHBwAM\n1peIiMgYmAghhKEnYWhXr15Fz549ER8fD3t7+xr3j4zcgIjwaITOmILg4EnPYIZERC+O2r4mvyh4\nK1UiIiIFYoATEREpEAOciIhIgRjgRERECsQAJyIiUiAGOBERkQIxwImIiBSIAU5ERKRADHAiIiIF\nYoATEREpEAOciIhIgRjgRERECsQAJyIiUiAGOBERkQIxwImIiBSIAU5ERKRADHAiIiIFYoATEREp\nULUCPDAwEG5ubvD09ISnpyf69Okj1cXFxcHX1xceHh4ICgpCXl6eVJeXl4cpU6bAw8MDvr6+iIuL\nk41rqL5ERERKV+0V+IIFC/Djjz/ixx9/xOHDhwEAaWlpWLBgAVasWIGkpCRYWlpi8eLFUp8lS5bA\n3NwcSUlJ+Pjjj7Fo0SKkpaUZtC8REZExqNUp9Li4OPTo0QMdOnSAlZUVQkJCcPToURQUFECv1+PI\nkSMICQmBlZUV2rdvjx49emDv3r0G7UtERGQMqh3gq1atQqdOnTB8+HCcOnUKQNlKWKPRSG0cHBxg\nbm6OzMxMZGZmwtTUFM2bN5fqXVxckJ6ebtC+RERExqBOdRqFhYXB0dERFhYWOHDgACZNmoS9e/dC\nr9fDxsZG1tba2ho6nQ5mZmYV6mxsbKDT6QDAYH2JiIiMQbUC3N3dXfrvgIAA7N+/H8ePH4dKpUJB\nQYGsbUFBAaysrGBqalplHQCD9SUiIjIGT/QZuImJCYQQcHZ2hlarlcpzcnLw4MEDNGvWDM2aNUNJ\nSQkyMzOleq1WCycnJwAwWF8iIiJj8NgAv3v3LhITE1FYWIji4mLs27cPZ86cQdeuXdG/f38cO3YM\nZ86cgV6vx5o1a+Dn5wdra2uoVCr4+fkhMjISer0eZ8+eRXx8PAYOHAgAButLRERkDB57Cr24uBir\nV6/Gr7/+CjMzM7Ro0QLR0dFo0aIFAGDx4sUICwtDXl4evLy8sHz5cqnvwoULMXfuXHTp0gX169fH\nokWL4OzsDKBsFW2IvkRERMbARAghDD0JQ7t69Sp69uyJ+Ph42Nvb17h/ZOQGRIRHI3TGFAQHT3oG\nMyQienHU9jX5RcFbqRIRESkQA5yIiEiBGOBEREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkREpEAM\ncCIiIgVigBMRESkQA5yIiEiBGOBEREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkREpEAMcCIiIgVi\ngBMRESkQA5yIiEiBahTgmZmZcHNzQ1hYmFQWFxcHX19feHh4ICgoCHl5eVJdXl4epkyZAg8PD/j6\n+iIuLk42nqH6EhERKV2NAnzJkiVwc3OTHqelpWHBggVYsWIFkpKSYGlpicWLF8vam5ubIykpCR9/\n/DEWLVqEtLQ0g/YlIiIyBtUO8AMHDsDGxgZeXl5SWVxcHHr06IEOHTrAysoKISEhOHr0KAoKCqDX\n63HkyBGEhITAysoK7du3R48ePbB3716D9iUiIjIG1QrwgoICREZGYvbs2bLytLQ0aDQa6bGDgwPM\nzc2RmZmJzMxMmJqaonnz5lK9i4sL0tPTDdqXiIjIGNSpTqPVq1fjrbfewiuvvCIr1+v1sLGxkZVZ\nW1tDp9PBzMysQp2NjQ10Op1B+xIRERmDxwZ4amoqkpOTERMTU6FOpVKhoKBAVlZQUAArKyuYmppW\nWWfIvkRERMbgsQF+6tQpXLt2Db6+vgDKVr8lJSUICAhAt27doNVqpbY5OTl48OABmjVrBlNTU5SU\nlCAzMxPNmjUDAGi1Wjg5OQEAnJ2dDdKXiIjIGDz2M/Bhw4bh6NGjiI2NRWxsLIYPH47u3bvj008/\nRf/+/XHs2DGcOXMGer0ea9asgZ+fH6ytraFSqeDn54fIyEjo9XqcPXsW8fHxGDhwIAAYrC8REZEx\neOwK3NLSEpaWltJjlUoFCwsLNGjQAA0aNMDixYsRFhaGvLw8eHl5Yfny5VLbhQsXYu7cuejSpQvq\n16+PRYsWwdnZGUDZKtoQfYmIiIyBiRBCGHoShnb16lX07NkT8fHxsLe3r3H/yMgNiAiPRuiMKQgO\nnvQMZkhE9OKo7Wvyi4K3UiUiIlIgBjgREZECMcCJiIgUiAFORESkQAxwIiIiBWKAExERKRADnIiI\nSIEY4ERERArEACciIlIgBjgREZECMcCJiIgUiAFORESkQAxwIiIiBWKAExERKRADnIiISIEY4ERE\nRArEACciIlKgagV4WFgYunbtirZt26JPnz7YvXu3VJecnAx/f3+4u7sjMDAQ165dk+qKioowZ84c\ntG3bFt7e3ti8ebNsXEP1JSIiUrpqBfi7776L7777DufOncO6deuwevVqXLp0Cbm5uZg6dSpCQkKQ\nkpICV1dXhIaGSv2ioqKQlZWFY8eOYevWrdi0aRMSEhIAwGB9iYiIjEG1AtzZ2RkWFhYAABMTE5iY\nmCA7OxtHjx6Fs7Mz+vbti7p162LatGnQarXIyMgAAMTGxiIoKAi2trZwdHTE0KFDERMTAwAG60tE\nRGQMqv0Z+KJFi+Du7o6+ffvCzs4Or7/+OtLS0qDRaKQ2KpUKDg4OSE9PR35+Pm7cuAEXFxep3sXF\nBenp6QBgsL5ERETGoEYBfu7cOXzxxRfw8/ODhYUF9Ho9bGxsZO2sra2h0+mg1+sBQFZvY2MDnU4H\nAAbrS0REZAxqdBW6mZkZ2rdvj+vXr+PLL7+ESqVCQUGBrI1Op4OVlRVUKhUAyOoLCgpgZWUFAAbr\nS0REZAye6GtkJSUlyM7OhrOzM7RarVSu1+uRnZ0NJycn2Nraws7OTlav1Wrh5OQEAAbrS0REZAwe\nG+C3b9/GgQMHoNPpUFJSgsTERBw4cACdO3eGn58f0tLScPjwYRQWFiI6OhoajQaOjo4AgEGDBmH9\n+vXIz89HRkYGdu/ejYCAAAAwWF8iIiJj8NgANzExwZdffonXX38dHTp0wIoVKzB37lz06tULDRo0\nQFRUFCIiItChQwdcuHAB4eHhUt/g4GA0bdoUvr6+CAwMxPjx4+Hj4wMAButLRERkDEyEEMLQkzC0\nq1evomfPnoiPj4e9vX2N+0dGbkBEeDRCZ0xBcPCkZzBDIqIXR21fk18UvJUqERGRAjHAiYiIFIgB\nTkREpEAMcCIiIgVigBMRESkQA5yIiEiBGOBEREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkREpEAM\ncCIiIgVigBMRESkQA5yIiEiBGOBEREQKxAAnIiJSIAY4ERGRAjHAiYiIFOixAV5UVIS5c+fC19cX\nnp6eGDRTJmLUAAAe8UlEQVRoEI4fPy7VJycnw9/fH+7u7ggMDMS1a9dkfefMmYO2bdvC29sbmzdv\nlo1tqL5ERERK99gALy4uxiuvvIJt27bh7NmzCAkJwfTp03H16lXk5uZi6tSpCAkJQUpKClxdXREa\nGir1jYqKQlZWFo4dO4atW7di06ZNSEhIAACD9SUiIjIGjw1wlUqFadOmwd7eHqampvD19YW9vT1+\n+uknHD16FM7Ozujbty/q1q2LadOmQavVIiMjAwAQGxuLoKAg2NrawtHREUOHDkVMTAwAGKwvERGR\nMajxZ+C3bt1CZmYmnJyckJaWBo1GI9WpVCo4ODggPT0d+fn5uHHjBlxcXKR6FxcXpKenA4DB+hIR\nERmDGgX4gwcPEBYWhoCAADg6OkKv18PGxkbWxtraGjqdDnq9HgBk9TY2NtDpdABgsL5ERETGoNoB\nXlpailmzZsHc3Bzz588HULbyLSgokLXT6XSwsrKCSqUCAFl9QUEBrKysDNqXiIjIGFQrwIUQmDdv\nHm7duoWoqCiYm5sDAJydnaHVaqV2er0e2dnZcHJygq2tLezs7GT1Wq0WTk5OBu1LRERkDKoV4AsX\nLkRGRgY2bNiAevXqSeV+fn5IS0vD4cOHUVhYiOjoaGg0Gjg6OgIABg0ahPXr1yM/Px8ZGRnYvXs3\nAgICDNqXiIjIGDw2wK9du4adO3ciNTUVXbt2haenJzw9PbFv3z40aNAAUVFRiIiIQIcOHXDhwgWE\nh4dLfYODg9G0aVP4+voiMDAQ48ePh4+PDwAYrC8REZExMBFCCENPwtCuXr2Knj17Ij4+Hvb29jXu\nHxm5ARHh0QidMQXBwZOewQyJiF4ctX1NflHwVqpEREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkRE\npEAMcCIiIgVigBMRESkQA5yIiEiBGOBEREQKxAAnIiJSIAY4ERGRAjHAiYiIFIgBTkT0HFu8eDFc\nXFyg0WjQqlWrStu0bdu2yrpyI0eOhEajqfG2NRoNjh07VqN+SnTw4EFoNBosX77c0FOptjqGngAR\n0f+FZcuWYefOnSgqKgIANG3aFN9++22FdhEREdi4cSNKS0sBAI0aNcKRI0dgaWkJADh06BBmzJiB\nkpISmJiYICwsDO+8847Uf+DAgUhPT8dPP/30VOa9Y8cOqFQqrFy5Eo0aNXricVatWoWsrKynMqen\nrV27drh37x5+/vlng82he/fu2LJly2PfCD1PuAInohdCXl4e6tevj27dulXZJj4+Hhs2bIClpSXm\nzZuH3r1748aNG/Dz85PazJo1C3Xq1MFHH30EW1tbrFy5Uqo7ePAgtFotIiMjn+rc3dzc0LNnT7i5\nuT3xGE2aNEGnTp2e4qyMR15eHiwtLeHl5QVbW1tDT6faGOBE9EJYuXIlEhMTsWnTpirbLFq0CABw\n8uRJjB49GlFRUWjVqhVu3ryJq1evAgCKiorg7++PgIAAhISEQAgh9Q8LC0ObNm3Qs2fPas1p165d\ncHV1hUajgUajgaurK2JiYgD8efoaAE6dOgWNRoORI0c+cryhQ4fKxjp16pRUV9kp9HHjxkntW7Zs\nidDQ0EpPIx89ehQtW7aU2oaFhcnqT506hbZt28rGWrZsmVSfn58PLy8vqV6j0cDd3R0A4Ovri4KC\nApSUlEh106dPr7BvGRkZ0Gg0GDt2rKw8MTERGo0GH3zwAQBg7Nix0kcO5c/D7t27pfblp8rHjBmD\n1q1bQ6PRoF+/fpWeQvfx8ZHNuUOHDkhLS6vwnH700UfSNl1cXLBmzRrZHPfu3Ys2bdpI47i4uCA0\nNFSqj46Oluai0WjQpk0bHDp0qMJz8FcMcCKi/+/27duwsrKChYWFVDZmzBgAwLZt2wAA9erVQ0JC\nAoqKirB9+3aYmpa9jP7973+HEAJfffVVtbZ1/fp1zJ8/H6WlpQgODkZwcDBKS0sxe/Zs5OXlISQk\nBFu2bAEAtGrVClu2bMGqVauqHK+kpARpaWmYO3cu3nnnHTx48ACTJk2qsn10dDSSkpLQpEkTLF26\nFL169cI333xTads9e/agf//+WL58OaysrBAXFyed7r5+/TpGjx6N4uJiTJ06FR999BHs7e2xbds2\n7NixAwAwYsQI5ObmYsSIEVi7di2CgoLQtGlTAMCWLVtgaWkJU1NTbNmyBVu2bMG8efMqzMHR0RGW\nlpZISUmRlf/jH/8AACxYsAAAUFhYCH9/f6xYsUIKyQ8++AB5eXmyfsnJyWjfvj1WrVqF2bNnV7rf\nFhYWePvtt7F69WoMGzYMd+/exd///vcK7T7//HOMHj0aixcvhpmZGdatW4eCggJpO7NmzUJxcTHG\njBmDFStWwM/PD2ZmZgCAL774ApGRkbCzs8PixYuluYSEhODmzZuVzksiSOTk5Ai1Wi1ycnKeqP+a\nNetFs9dcxZo165/yzIjoWVCr1aJnz56Vlnft2lVWVv76MHHiRCGEEAkJCcLV1VWo1WrRsmVL8eWX\nX4rvv/9eqNVqERcXJ3r16iXUarXQaDQiPDy8yjn8/e9/F2q1Wpw7d04qO3nypFCr1WLUqFGyOY0e\nPfqR++Pp6SnUarW4ffu2VNa7d2+hVqulxyNGjJA9dnd3Fy4uLrJxBg8eLNRqtfjwww+FEEIsWrRI\nqNVqMXny5ApznDVrlmzcP/74QzZWy5YtRadOnYQQQnTq1Em0bNlSFBcXVzr/tm3bipYtW0qPq3pN\nfu+994RarRY//PCDVKbRaCocs4dduHBBqNVqsXz5ciGEEN98841Qq9WiR48esnbl5eX7XpmgoCCh\nVqtFYWGhbN+jo6OlNuHh4UKtVoudO3cKIYTw8fF5ZL54enqKNm3ayMpu374t1Gq1mDp1apVzEUKI\naq3At2/fjsGDB8PV1bXCO5Xk5GT4+/vD3d0dgYGBuHbtmlRXVFSEOXPmoG3btvD29sbmzZufi75E\nRNVVvsI2MTEBAHTr1g0XL17EL7/8gp9//hnDhw/HlClT4OLigu+//x7Z2dnYt28f3nzzTWzYsKHC\nyq9cdnY2TExM4OnpKZV16tQJJiYmuHLlyhPNs0GDBtLjxo0bP7L9/fv3Ze0BoG/fvpW2fbi8/HP0\n3377DQCkU8rt2rWTnW4uKSmRVqHjxo1DSUkJWrVqhQ4dOmDy5MlVPi+PsnjxYgCQTs+vW7cOQghM\nnjxZarNx40a4ublJ8xgyZAgA4JdffpGN1b59+8duLygoSPbRQflFjw9/NAFAdlq/TZs2AIDLly8D\nAG7duoW6devC3t6+0m3o9Xrcv39f9tx5eXkBANLT0x85v2pdhd6oUSMEBQUhMTERhYWFUnlubi6m\nTp2KZcuWoUePHli9ejVCQ0Oxa9cuAEBUVBSysrJw7Ngx3Lp1C6NHj4ajoyN8fHwM1peIqCpmZmbQ\n6XSysrNnzwIAXnvttUr7jB49GiUlJdizZw+6du2Khg0bQqPRYNWqVdi/fz9iYmIqfG77LJS/wajq\ncW1YW1tXKCu/Sl8IARMTk0q/ftWwYUMAwMSJE9GnTx+sWLECZ86cwXfffYdOnTrhm2++gaOjY7Xn\nYWlpiYYNGyIjIwMAsHXrVpiYmGDEiBEAyt5UrFq1CnXr1sWIESPg7OwMoCz4y799UK5+/fqP3FZE\nRATi4+PRpEkTDBo0CPb29ti3bx9SUlKkNyYPz6tc+Ru+kpKSau2TEALW1tbSZ/gPe9zX/qoV4L17\n9wYAXLx4Eb///rtUfvToUTg7O0vvzqZNm4bOnTsjIyMDjo6OiI2NxfLly2FrawtbW1sMHToUMTEx\n8PHxMVhfIqKqvPzyy7hx4waKioqkz8E/++wzAEBgYGCF9qdOncKpU6ewbNkymJmZQQghvXD/9d9/\n5eDggNzcXPz444/SKvzUqVMQQqBFixZPfd/+ql69esjNzZWVVefCqb9q0aIFzp8/D3t7e3To0KHK\ndq+99hqio6MBlJ1BHTNmDKKiorB69WqYmprKLgZ8lIkTJ+LDDz/EunXrcOfOHdnXvsovAJw9e7YU\n6itWrKjxPgFl+QYAx48fl8o2btxY43EaNmyI69ev4+rVq5WuwlUqFXQ6HQYMGCB9Ll5dtbqILS0t\nTfYOQaVSwcHBAenp6cjPz8eNGzfg4uIi1bu4uEinBAzVl4heTFlZWdi4caP0Inz37l1s3LhRetEH\n/rwK3cvLC1u3bkVISAh+/vln2NnZVfriO2HCBLRo0QJDhw4FUHZDlTt37iA6OhqDBg0CUPa98MqU\nX6k8cuRIrFu3DtHR0dJK/Wl/Da0yEyZMQGlpKbp3747du3cjJCQEFy9eBFCz1fvatWthYmKCwMBA\nzJkzBwcPHkRERAR69+6NadOmASg7BT9mzBhs3boV+/fvly4469KlC4CykCstLcXWrVvx448/4s6d\nO1Vu7+2334aJiYn0HJWfVn94vIiICOzduxcLFiyQ3oDVVHnGDBs2DHFxcQgMDER2dnaNxym/yK53\n79746KOPsH//foSGhkpX8r/33nsQQqBdu3ZYvXo19u/fj0WLFqFz586Ii4t75Ni1upGLXq+v8BmK\ntbU1dDod9Ho9AMDGxkaqs7GxkU5PGaovEb2YtmzZIl0VDZR9tWnVqlUwMzNDQEAAAKBnz56YOHEi\nNm3aJL3wlt/I5a8mT56MoqIi7NmzRyqLiopCt27dpHAZNmwY7OzsKp1P+dXfixcvlsLc3Nwc//jH\nPx57evdpmDJlCs6ePYukpCR88MEHMDU1hY+PDxISEmSvn49jZ2eHXbt2Yfz48dizZ4/0fFhYWGDw\n4MEAACsrK5w8eRLJyckAyt4g9O7dW7qie8WKFRg+fLj0nD/u++7Ozs64fPkyLCwspM+cAcDT0xO+\nvr44duwYZs2aJZ1e/+KLL6r/xPx/EREROHfuHM6fP4/z58+jTp068Pb2RlJSUo3G6dq1K5YtW4Yl\nS5ZI12OZmJigX79+AMqu0C8sLER4eDjWr18v9bOxsUGzZs0ePfgjL3H7i/DwcPH+++9Lj5cuXSoW\nLlwoa9OvXz9x6NAhkZeXJ9Rqtbh165ZUd+jQIdGvXz+D9q0Mr0InIhJi1KhRQq1WC61Wa9B51PY1\n+UVRq1Pozs7O0Gq10mO9Xo/s7Gw4OTnB1tYWdnZ2snqtVgsnJyeD9iUiojJ9+/ZFdHQ09u/fjwkT\nJiAlJQX16tWr8T3TyTCqFeDFxcUoLCxEaWkpSkpKUFhYiOLiYvj5+SEtLQ2HDx9GYWEhoqOjodFo\npKsKBw0ahPXr1yM/Px8ZGRnYvXu3dKrKUH2JiKjM9evXERkZiZkzZyIhIQENGzbEwYMHDT0tqq7q\nLNMjIyOFWq2W/RMZGSmEECIpKUn06dNHuLm5iVGjRslOeRQWForZs2cLT09P4eXlJT777DPZuIbq\n+1c8hU5E9PzgKfTqMRGimtfuG7GrV6+iZ8+eiI+Pr/LL9o8SGbkBH69fDY/hLdGxe1upvPlLzTGq\nzSgUFBUgIjmiQr8pHaeggWUD7Li4Axm5GbK6dn9rhzec38DlW5ex86edsro6pnUwp9scAMDalLW4\nc09+xWYfxz7oaN8RSdlJ+O7Kd7K6RlaN8G77dwEAS48vrTCnUW1GoflLzbFPuw//+f0/sjqXhi4Y\n2noorhdcxydnP6nQ9/2u78PCzAJbftyCnLs5sjpvB2/0aN4DF36/gL3avbI6KwsrzPCaAQAITw6H\nrkh+weFAl4Fo07gNvrvyHZKy5ReQNP2fphjjOQZFJUX454l/VpjThHYT0MS6CXb/tBvaW1pZnXtj\ndwxwGYArd65g+4XtFfrOf30+AOBfZ/6FG7obsroezXvA28EbKVdTcDjjsKzuJcuXMLXjVADA8sTl\nKC4tltUPaz0M6oZqfJP2Dc7+dlZW59jAESPcRiD3Xi6iU6IrzCnUKxTWFtbYfmE7rtyR3+yj46sd\n0cepD1JvpuLfP/9bVmdhZoH3u74PAIg6FYW8+/KbaLzh/Aba/a0dEjITcDzruKyuiXUTTGg3AUDl\nfzNve7wNB1sHxKTG4NKNS7K6Vnat8Fart3At/xo+O1/xauC5XefCzMwMn577FL/98ZusrptDN3Rv\n3h3n/3secZflV+Pa1LXB9M5l98te9cMq6B/oZfUBLQPg2sgVRzOO4uTVk7K612xfw2iP0bhXdA8r\nk1firya1mwQ7azvsvLgTl3Mvy+o8X/FEP3U/ZORmYMfFHbI6ExMTfOBT9l3e9SnrceveLVm9Xws/\ndG7aGSdzTuLor0dldQ0tG2Jyx7KbkSxLWFbhK1Uj3EbAsYEj9l/ejx//+6OsTt1AjWFuw3Cz4CY2\nnN1QYX/CvMJgaWGJree3Iitf/mtkne07w8/RD5duXEJMaoysTmWuwswuMwEAq0+uxh+Ff8jq+6v7\nw+MVD3x/5XskZifK6sr/Fp9UbV+TXxQMcDydAF/w3QLke8rvW1vXrC7uf3AfJ3NOwuszrwr9Do08\nhD5OfaD6hwr3iu/J6l61eRVXZ1zF9EPTsebUmgp9xcKyw2a62BQC8kPYo1kPxL8dj06bOiHlmvy+\nwaYmpihZUPa9VJPFFb8qsqT7Esx/fT4afdwIN/Xy/bGxsMHdOXex9fxWvL337Qp9s6ZnwcHWARZL\nLfCg9IGsTvOyBtqpWgzbPQy7ft5V5f5UNqe/t/o7dg7dCZe1LvjltvxuSuam5iiaX4Ts/Gy8trri\njTY+H/g5RnuMxv8s/x/8USR/AbJT2eHGezew9PhSLPh+QZVzMltihlJRKqvr+GpHnHrnFHp+3hPf\nZcrfJJnABKULS6vcn5BOIVjtvxr24fa49sc1WZ1lHUvo5+lxOP0w/L/wr9A3eVwyOjftjHrL6qGw\npFBW95rta8icnomJcRPxybmKb7Ae9Rz7O/rj4KiD8PyXJ85fPy+rMzMxQ/GC4ir7rvRbiZldZuLl\nFS8j9578e8W2dW2RNzsP61PWI+hgUIW+ebPyYGtpC/Ol5hXe6LjaueJi0EUM+moQ9v6yt0LfR+3P\n6Daj8XnA53CKdELGHfmbYwszCxR+UIhLNy7BbX3Fq53/PfTfeKvVW7D+0Bq6B/I3k02smuC/Yf/F\nnPg5+OjER1XOqbL/L7s27YrEcYno9lk3nMg5Iat73N/M7K6zsbzncryy8hVc112X1VmZW6FgbgG+\n/vlrDNk9pELfi5MvwrWRK+ouq4uiEvnNTBxfckR6cDrejnkbWy9srXJ/KpvTQM1AxA6Phds6N1y6\nKX/jVv63+KQY4NXD3wN/Smx+aYBunt4VVuAA4NrYFUu6L6nQp8OrZTc92DRgU6UrcAAIah+Ely1f\nltXVMf3zsEX2jax0BQ4A4b3DK12Bl6tsTqPajCqbU/9Nla7AAaC3U+9K+zaxbgIA2NhvY6UrcACY\n5zMPro1cZXVWFlbSf6/qvarSFTgArHtzXaUr8PJtVzan3k5lNyH6dMCnla7AH97nqqx7Y12lK3AA\nWN5zeaUr8HIf9viw0hU4AGzsv7HSFThQ9rdR2f64Ni577jYN2FTpChwAQjuHSs9LOQuzP3+cI9I/\nstIVOACs6bOm0hV4ucrmNLR12XegN/XfVOkKHAAGaAZUWJECgLVF2R2+Nry5odIVOAAsen0R2r3S\nTlZnU/fPrzmt9FtZ6QocANa/ub7SFTgAONZ3rHR/fBzKbvj0af9PK12BA8A7nu9AVUclq3v4u9PR\nfaMrXYEDwMd+H1e6Ai+31HdppStwAPhkwCeVrsDL513Z/jjWL/ub+qTfJ5WuwAHgPe/34NRAfqGv\nyvzP/YvoE1HpChwAovpGVboCp2ePK3A8nRV4RHg0QmdMQXBw1b/+Q0REj8cVePXw50SJiIgUiAFO\nRESkQAxwIiIiBWKAExERKRADnIiISIEY4ERERArEACciIlIgBjgREZECMcCJiIgUiAFORESkQAxw\nIiIiBWKAExERKRADnIiISIEY4ERERArEACciIlIgow3wvLw8TJkyBR4eHvD19UVcXJyhpwQASDpx\nCjNC5yLpxCmDt1fq2M/TXLifz/fYz9NcuJ/0tJkIIYShJ/EszJgxA6WlpfjHP/6B1NRUvPvuu/jq\nq6/g7OxcoW1tfzw+MnIDIsKjETpjCoKDJ1XZrq//W9BqL0uPXVzUOHjoa4O0V+rYz9NcuJ/P99jP\n01y4nzVT29fkF0UdQ0/gWdDr9Thy5Aji4uJgZWWF9u3bo0ePHti7dy/CwsKeyTZtbG1xR1+E4yn/\nqbQ+Iy0DN3Pz8XKjxlLZzdx8bN4RixZOzf9P2yt17OdpLtzP53vs52kuL+J+FhXexx/5+dBqLyPp\nxCl4d+1UYWyqPaNcgf/8888YPnw4Lly4IJV9+umnOH36NDZs2FChfW3f7X2+fTd+vVUAU1Oj/USC\niKjaSktLsfNf61F47x4CAvojPOLDGvXnCrx6jHYFbmNjIyuzsbGBTqd7Jtt7e9RQfB13BA0bN66y\nza/pVxC9em2F8inTp1b6zvdZtlfq2M/TXLifz/fYz9NcXsT9LCq8j8J79wAAb701sEJ/ejqMdgX+\nv//7v/jPf/48nf3ZZ58hJSXlmazAq+uvnw+1bKnGNwer/9nT02yv1LGfp7lwP5/vsZ+nuXA/a4Yr\n8GoSRkin04nWrVuLK1euSGXvvfee+Pjjjyttn5OTI9RqtcjJyXnmczuReFKETp8jTiSeNHh7pY79\nPM2F+/l8j/08zYX7WX3/l6/JSmaUK3AACA0NhYmJCZYtW4bU1FRMnDjxmV2FTkRETw9fk6vHaK+6\nWrhwIe7fv48uXbpg5syZWLRoUaXhTUREpERGeREbANSvXx/r1q0z9DSIiIieCaNdgRMRERkzBjgR\nEZECMcCJiIgUyGg/A6+JkpISAMD169cNPBMiIip/LS5/babKMcAB3Lx5EwAwcuRIA8+EiIjK3bx5\nE6+99pqhp/HcMtrvgdfE/fv3cenSJdjZ2cHMzMzQ0yEieqGVlJTg5s2bcHV1Rb169Qw9necWA5yI\niEiBeBEbERGRAjHAiYiIFIgBTkREpEAMcCIiIgVigNdSXl4epkyZAg8PD/j6+iIuLs7QU3omAgMD\n4ebmBk9PT3h6eqJPnz6GnlKtbd++HYMHD4arqytmz54tq0tOToa/vz/c3d0RGBiIa9euGWiWtVfV\nfl69ehUajUY6pp6enoiOjjbgTGunqKgIc+fOha+vLzw9PTFo0CAcP35cqjeWY/qo/TS2Y0qPxu+B\n19KSJUtgbm6OpKQkpKam4t1334WLi4tR/vLZggULMHToUENP46lp1KgRgoKCkJiYiMLCQqk8NzcX\nU6dOxbJly9CjRw+sXr0aoaGh2LVrlwFn++Sq2s9yp0+fRp06yn8pKC4uxiuvvIJt27bhb3/7G44f\nP47p06cjLi4OKpXKaI7po/aznLEcU3o0rsBrQa/X48iRIwgJCYGVlRXat2+PHj16YO/evYaeGlVD\n79690atXL9SvX19WfvToUTg7O6Nv376oW7cupk2bBq1Wi4yMDAPNtHaq2k9jo1KpMG3aNNjb28PU\n1BS+vr6wt7fHTz/9ZFTH9FH7SS8WBngtZGZmwtTUFM2bN5fKXFxckJ6ebsBZPTurVq1Cp06dMHz4\ncJw6dcrQ03lm0tLSoNFopMcqlQoODg5Ge1x9fX3h4+ODOXPmIDc319DTeWpu3bqFzMxMODk5GfUx\nfXg/yxnrMSU5Bngt6PV62NjYyMpsbGyg0+kMNKNnJywsDN9++y0SExMxbNgwTJo0CdnZ2Yae1jNR\n2XG1trY2uuP60ksv4d///jeOHTuGPXv2QKfT4b333jP0tJ6KBw8eICwsDAEBAXB0dDTaY/rX/TTm\nY0oVMcBrQaVSoaCgQFZWUFAAKysrA83o2XF3d4e1tTUsLCwQEBCAtm3byi4QMiaVHVedTmd0x9XK\nygpubm6oU6cOGjZsiPnz5+PEiRMV9l1pSktLMWvWLJibm2P+/PkAjPOYVrafxnpMqXIM8Fpo1qwZ\nSkpKkJmZKZVptVrZqSxjZWJiAmO9C6+zszO0Wq30WK/XIzs72+iPq4mJCQAo+rgKITBv3jzcunUL\nUVFRMDc3B2B8x7Sq/fwrYzimVDUGeC2oVCr4+fkhMjISer0eZ8+eRXx8PAYOHGjoqT1Vd+/ela5g\nLi4uxr59+3DmzBl07drV0FOrleLiYhQWFqK0tBQlJSXS/vn5+SEtLQ2HDx9GYWEhoqOjodFo4Ojo\naOgpP5Gq9vM///kPfv31V5SWluLOnTtYtmwZOnbsWOFUs5IsXLgQGRkZ2LBhg+xHMIztmFa1n8Z4\nTKlq/DGTWsrLy8PcuXPxww8/oH79+pg5cyb69+9v6Gk9Vbm5uZgwYQJ+/fVXmJmZoUWLFggJCYG3\nt7ehp1YrUVFRWLt2raxs6tSpmDZtGn744QcsWbIEv/32G9zd3bF8+XLY29sbaKa1U9V+Nm/eHOHh\n4cjNzYW1tTW6dOmC9957D3Z2dgaaae1cu3YNPXr0gIWFhewrVIsXL8aAAQOM5pg+aj9NTU2N6pjS\nozHAiYiIFIin0ImIiBSIAU5ERKRADHAiIiIFYoATEREpEAOciIhIgRjgRERECsQAJyIiUiAGOBER\nkQIxwImIiBTo/wHDHGskXHm80AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11f408750>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEhCAYAAABsu5d6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHeZJREFUeJzt3X1UVXXa//GPKGAgiYyamk6aIFiIECdRNBMUs0dUnMqn\nCh+yMrOkUnMlLKqxcirDSctsxCkzG03QnMqyWzObSAyXmfiAM5hooqg4Ho4gHPbvj265QyC+6uHB\nfu/XWqzV+Z5r73Pta7H4tM8+Z9vEsixLAADUwq2hGwAAXB4IDACAEQIDAGCEwAAAGCEwAABGCAwA\ngBECA2ikZsyYoQceeKCh26hRXl6eAgMDlZmZ2dCtoJ4QGHCZGTNmKDAwsMrPunXrXPo61113nT76\n6COX7vP/FzExMZo/f35Dt4HLVLOGbgC/LzabTfPmzau0duWVVzZQN7UrLS2Vu7t7Q7cBXBY4w4BL\nubu7q02bNpV+PD09K55ft26dYmNj1aNHD0VHR2vOnDlyOBwVz2/ZskVjx45Vr169FB4erjFjxmjH\njh0Vz0dHR8vpdGrmzJkVZzCS9NFHH+m6666r1MuRI0cUGBiojIwMSVJGRoYCAwO1ceNGjRw5Uj16\n9NA//vEPSdLOnTs1btw4hYWFqXfv3nr00Ud16NChSvuaMmWKIiIiFBISooEDB2rx4sU1zqG0tFRz\n5sxR//79FRwcrH79+umJJ56oVFPbLKpjss2yZct02223KTg4WH369NFjjz0mSRo7dqx++ukn/fWv\nf62YXV5eniTpwIEDmjJlimw2m2688UaNGzdOe/bsqbTff/7zn4qJiVGPHj107733Vnkev3+cYaDe\nfPTRR5ozZ45mzZql8PBwHTlyRMnJyTpx4oTmzp0rSXI4HBo5cqSCgoLkdDqVmpqqCRMm6LPPPlOr\nVq20cuVK9evXT9OnT9dtt912UX28+OKLevrppxUQECB3d3fl5ORo7Nixio+P16xZs1RWVqY33nhD\n48aN05o1a+Tp6amkpCQVFxcrNTVVPj4+ysvLU0FBQY2v8d577+mTTz7R3Llz1alTJxUUFOj777+/\noFlczPxSUlK0ZMkSJSQkqG/fvnI4HNq0aZMkaf78+Ro+fLhuueUWjRs3TpLk5+engoICjRo1SoMG\nDdKyZcvk7u6uZcuW6b777tMnn3wiPz8/7dq1S9OmTdPEiRM1bNgw5eTk6IUXXrio+eMyZgEuMn36\ndKt79+5WaGhoxc/gwYMrno+KirLef//9Stt89913Vrdu3azCwsJq9+l0Oi2bzWalp6dXrHXv3t1a\ntWpVpbpVq1ZZ3bt3r7T2888/W926dbO+/fZby7Is69tvv7W6detmrV69ukrfjz/+eKW1kpISKyQk\nxPr8888ty7KsO++800pJSTEZg2VZlvXcc89ZY8eOtcrLy6t93mQW06dPt+6//37jbYqKiqwePXpY\nixcvrrGvQYMGVTmOlJQU609/+lOltfLycmvgwIHWkiVLLMuyrISEBOuee+6pVPPuu+9a3bp1s7Zu\n3Vrj6+H3hTMMuFRISIheeumlisdNmzaVJJ04cUKHDh3Siy++qJdffrnieet/73154MABhYSE6ODB\ng0pJSdH27dt1/PhxWZalM2fO6PDhwy7t8dd++OEHHThwQGFhYZXWS0pKlJubK0m6//77lZiYqK++\n+kq9evXSgAEDdOONN9b4GnFxcYqPj1dMTIwiIyPVt29fRUVFycPDw3gWv2ayzbme+/bte0Hz+OGH\nH/Tjjz9WOf7i4uKK/e7fv1+9e/eu9Hx4ePgFvQ4ufwQGXKp58+a65pprqqyXl5dLkmbNmqWIiIgq\nz7dr106S9NBDD6lVq1aaPXu22rdvL3d3d40aNUqlpaW/+bpublUvx9W0jZeXV5XeYmNj9eCDD1ap\n9fX1lfRLANx0003avHmzMjIyNHHiRA0aNEh/+ctfqn2N7t27a8OGDfrmm2+UkZGhF154Qa+//ro+\n/PBD41mc32Nt25y7ptCkSZNqe6pJeXm5evfurdmzZ1d5zsfHR9IvwXSh+8XvD4GBetG6dWu1b99e\n//nPf3T33XdXW3Py5Enl5ORo0aJFuummmyT9crH5+PHjlerc3d3ldDorrfn5+cnpdKqgoECtW7eW\nJO3atcuot+DgYO3Zs0d//OMff/OPYtu2bRUXF6e4uDjdfPPNmjZtmpKSktSiRYtq6729vRUTE6OY\nmBhNmjRJ/fr103fffafo6OhaZ3E+k/l17dpVnp6e+vrrrys+DHC+6mYXHBys1atX66qrrlLz5s2r\n3c7f37/SNRhJVR7j969pUlJSUkM3gd+HL774Qna7XUOHDq32eV9fX82bN09NmjRRy5YtderUKW3b\ntk2pqamKioqSp6enli9frpMnT6p79+7Kzc1VYmKiTp48qRtuuKHi/6w//vhjlZSUqGfPnjp79qyu\nuOIKtWzZUsuXL9fRo0fl7++vHTt26PXXX9fx48c1bNgwdezYUYcOHdLq1asVHx9f6Y98165d9dZb\nb2nfvn1q3769zp49q+zsbKWmpqpLly5q2bKlkpOTJf1yJnP06FEtX75cRUVFmjhxYrUhs3jxYuXl\n5cnNzU0Oh0Pp6enaunWrHnnkEbVq1arWWVQ3z9q28fDwUHFxsd5++235+PjIx8dH+fn5WrVqlWw2\nmyRp06ZNOnjwoCIiIlRaWipPT08FBARo+fLlysjI0NVXX63y8nLt3btXK1askKenp9q3b68OHToo\nJSVFpaWlateunTIzMzV//nz997//VVxcnDp06OC6XyQ0WpxhoN4MHTpULVq00Ntvv60333xTTZs2\nVadOnRQTEyPplz/Gr7/+up5//nnddddd6tChg6ZNm1blbZ/p06drzpw5GjhwoEpLS7Vnzx75+vrq\n1Vdf1UsvvaS77rpL1113nZ566ilNmDCh1r66du2qDz74QPPmzdP48eNVUlKiq666Sr179670lsyf\n//xn/fzzz7riiivUs2dPvf322zWekbRo0UKpqanKzc2VZVm69tprlZKSomuvvdZoFhczP0l6/PHH\n5efnp7///e+aM2eOrrzyyoqwkKQpU6Zo9uzZGjJkiEpKSrRhwwZ17NhRK1as0KuvvqpHH31Udrtd\nbdq0UXh4uNq0aSPpl7OQV155Ra+99preeecdde/eXTNnztTkyZNrnS9+P5pYFv/iHgCgdnxxDwBg\nhMAAABghMAAARggMAICRBv2UVHFxsXbu3Kk2bdpUfCMYANAwnE6njh07puDg4Gq/k9OggbFz506N\nHj26IVsAAJxn2bJllT6OfU6DBsa5z3gvW7as2tshAADqz5EjRzR69OiKv83na9DAOPc2VLt27dSx\nY8eGbAUA8L9qukTARW8AgBECAwBghMAAABghMAAARggMAIARo8B47733NHz4cAUHB2vGjBm/WZua\nmqq+ffsqPDxcM2fO1NmzZ13SKACgYRkFRtu2bfXII48oLi7uN+s2b96sRYsWKTU1VV9++aXy8vKU\nkpLikkYBAA3LKDAGDx6sQYMGVfz7xjVJS0vTiBEjFBAQoJYtW+qRRx7R6tWrXdIoAKBhufSLe/v2\n7dPAgQMrHgcGBqqgoEAnT55Uq1atXPlSQIOJihplXPs///N+HXYC1C+XXvR2OByV/q3kc/+8ZVFR\nkStfBgDQAFwaGF5eXrLb7RWPz/23t7e3K18GANAAXBoYAQEB2rNnT8Xj3bt3q3Xr1rwdBQC/A0aB\nUVZWppKSEpWXl8vpdKqkpERlZWVV6mJjY7Vy5Url5OTo1KlTWrhwoYYNG+bypgEA9c8oMBYuXKiQ\nkBAtWrRIa9asUUhIiBYuXKjDhw8rLCxMhw8fliT1799fEyZM0H333aeoqChdffXVeuyxx+r0AAAA\n9cPoU1JTpkzRlClTqn0uKyur0uP4+HjFx8dfemcAgEaFW4MAAIwQGAAAIwQGAMAIgQEAMEJgAACM\nEBgAACMEBgDACIEBADBCYAAAjBAYAAAjBAYAwAiBAQAwQmAAAIwQGAAAIwQGAMAIgQEAMEJgAACM\nEBgAACMEBgDACIEBADBCYAAAjBAYAAAjBAYAwAiBAQAwQmAAAIwQGAAAIwQGAMAIgQEAMEJgAACM\nEBgAACMEBgDACIEBADBCYAAAjBAYAAAjBAYAwIhRYBQWFmry5MkKDQ1VVFSU1q5dW23d2bNnNXv2\nbEVGRqpXr1566KGHlJ+f79KGAQANwygwkpOT5e7uri1btmju3LlKSkrSvn37qtQtXbpU27dv15o1\na7R582b5+Pjoueeec3nTAID6V2tgOBwOrV+/XlOnTpW3t7dsNpuio6OVnp5epTYvL0/9+vVT69at\n5enpqdtvv73aYAEAXH5qDYzc3Fy5ubmpS5cuFWtBQUHKycmpUjtixAh9//33ys/P15kzZ7R27Vr1\n79/ftR0DABpEs9oKHA6HfHx8Kq35+PioqKioSm2XLl3UoUMH9e/fX02bNlW3bt307LPPuq5bAECD\nqfUMw8vLS3a7vdKa3W6Xt7d3ldrExESVlJQoIyND27dvV0xMjCZOnOi6bgEADabWwOjcubOcTqdy\nc3Mr1nbv3i1/f/8qtXv27NGwYcPk6+srDw8PjR07Vjt27NCJEydc2jQAoP4ZnWHExMQoJSVFDodD\n27Zt04YNGxQbG1ultkePHkpPT9fp06dVWlqq999/X23btpWfn1+dNA8AqD9GH6tNTExUcXGxIiMj\nlZCQoKSkJAUEBCgzM1NhYWEVdU8//bQ8PDw0ePBg9enTR5s2bdIbb7xRZ80DAOpPrRe9JcnX11cL\nFiyosm6z2ZSVlVXxuFWrVnrllVdc1x0AoNHg1iAAACMEBgDACIEBADBCYAAAjBAYAAAjBAYAwAiB\nAQAwQmAAAIwQGAAAIwQGAMAIgQEAMEJgAACMEBgAACMEBgDACIEBADBCYAAAjBAYAAAjBAYAwAiB\nAQAwQmAAAIwQGAAAIwQGAMAIgQEAMEJgAACMEBgAACMEBgDACIEBADBCYAAAjBAYAAAjBAYAwAiB\nAQAwQmAAAIwQGAAAIwQGAMAIgQEAMEJgAACMGAVGYWGhJk+erNDQUEVFRWnt2rU11v74448aPXq0\nwsLCFBkZqaVLl7qsWQBAw2lmUpScnCx3d3dt2bJF2dnZmjRpkoKCghQQEFCp7sSJE5owYYJmzpyp\nIUOG6OzZs8rPz6+TxgEA9avWMwyHw6H169dr6tSp8vb2ls1mU3R0tNLT06vUpqamql+/frrrrrvk\n4eGhFi1aqGvXrnXSOACgftUaGLm5uXJzc1OXLl0q1oKCgpSTk1Oldvv27WrZsqXuvfde9enTRw89\n9JAOHz7s2o4BAA3C6AzDx8en0pqPj4+Kioqq1Obn5ystLU3PPPOMNm7cqI4dO2ratGmu6xYA0GBq\nDQwvLy/Z7fZKa3a7Xd7e3lVqPT09FRMTo5CQEHl6emry5MnKysrS6dOnXdcxAKBB1BoYnTt3ltPp\nVG5ubsXa7t275e/vX6U2MDCw0uMmTZpIkizLusQ2AQANzegMIyYmRikpKXI4HNq2bZs2bNig2NjY\nKrXDhw/XF198oezsbJWWlmrBggUKDw/XlVdeWSfNAwDqj9H3MBITE1VcXKzIyEglJCQoKSlJAQEB\nyszMVFhYWEVdnz599MQTT+jBBx9UZGSkfvrpJ73yyit11jwAoP4YfQ/D19dXCxYsqLJus9mUlZVV\naW3UqFEaNWqUa7oDADQa3BoEAGCEwAAAGCEwAABGCAwAgBECAwBghMAAABghMAAARggMAIARAgMA\nYITAAAAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBghMAAABghMAAARggMAIARAgMA\nYITAAAAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBghMAAABghMAAARggMAIARAgMA\nYITAAAAYITAAAEaMAqOwsFCTJ09WaGiooqKitHbt2t+sP3v2rIYMGaL+/fu7pEkAQMNrZlKUnJws\nd3d3bdmyRdnZ2Zo0aZKCgoIUEBBQbf0777yjP/zhD3I4HC5tFgDQcGo9w3A4HFq/fr2mTp0qb29v\n2Ww2RUdHKz09vdr6gwcPas2aNXrwwQdd3iwAoOHUGhi5ublyc3NTly5dKtaCgoKUk5NTbf3zzz+v\nadOmqXnz5q7rEgDQ4IzOMHx8fCqt+fj4qKioqErt559/rrKyMsXExLiuQwBAo1DrNQwvLy/Z7fZK\na3a7Xd7e3pXWHA6H5s6dq0WLFrm2QwBAo1BrYHTu3FlOp1O5ubnq3LmzJGn37t3y9/evVHfgwAEd\nOnRIo0ePliSVlpbq9OnT6tu3r1asWKGOHTu6vnsAQL0xOsOIiYlRSkqKnn/+eWVnZ2vDhg364IMP\nKtUFBARo48aNFY+zsrKUnJys1atXy8/Pz+WNAwDql9H3MBITE1VcXKzIyEglJCQoKSlJAQEByszM\nVFhYmCSpWbNmatOmTcVPy5Yt5ebmpjZt2qhp06Z1ehAAgLpn9D0MX19fLViwoMq6zWZTVlZWtdtE\nREToq6++urTuAACNBrcGAQAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBghMAAABgh\nMAAARggMAIARAgMAYITAAAAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBghMAAABgh\nMAAARggMAIARAgMAYITAAAAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBghMAAABgh\nMAAARggMAIARAgMAYMQoMAoLCzV58mSFhoYqKipKa9eurbZu8eLFuuOOOxQWFqbo6GgtXrzYpc0C\nABpOM5Oi5ORkubu7a8uWLcrOztakSZMUFBSkgICASnWWZemll15SYGCgfvrpJ40fP17t27fX7bff\nXifNAwDqT61nGA6HQ+vXr9fUqVPl7e0tm82m6OhopaenV6mdOHGirr/+ejVr1kzXXnutBg4cqO+/\n/75OGgcA1K9aAyM3N1dubm7q0qVLxVpQUJBycnJ+czvLspSZmSl/f/9L7xIA0OCMzjB8fHwqrfn4\n+KioqOg3t5s/f77Ky8sVFxd3aR0CABqFWq9heHl5yW63V1qz2+3y9vaucZv33ntPaWlpev/99+Xh\n4XHpXQIAGlytZxidO3eW0+lUbm5uxdru3btrfKtp5cqVWrRokZYuXap27dq5rFEAQMOqNTC8vLwU\nExOjlJQUORwObdu2TRs2bFBsbGyV2jVr1ui1117TkiVL1KlTpzppGADQMIy+h5GYmKji4mJFRkYq\nISFBSUlJCggIUGZmpsLCwirq5s2bp8LCQo0YMUJhYWEKCwvT7Nmz66x5AED9Mfoehq+vrxYsWFBl\n3WazKSsrq+Lxl19+6brOAACNCrcGAQAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBg\nhMAAABghMAAARggMAIARAgMAYITAAAAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBg\nhMAAABghMAAARggMAIARAgMAYITAAAAYITAAAEYIDACAEQIDAGCEwAAAGCEwAABGCAwAgBECAwBg\nhMAAABghMAAARggMAIARo8AoLCzU5MmTFRoaqqioKK1du7baOsuyNHfuXEVERCgiIkIvv/yyLMty\nacMAgIbRzKQoOTlZ7u7u2rJli7KzszVp0iQFBQUpICCgUt2KFSv0xRdfKD09XU2aNFF8fLw6deqk\nkSNH1knzAID6U+sZhsPh0Pr16zV16lR5e3vLZrMpOjpa6enpVWrT0tI0btw4tWvXTldddZXi4+O1\nevXqOmkcAFC/aj3DyM3NlZubm7p06VKxFhQUpK1bt1ap3bdvn4KCgirV7du3r8Z9O51OSdKRI0cu\nqGmgIZWVlRjX5uXl1WEngGud+1t87m/z+WoNDIfDIR8fn0prPj4+Kioqqra2RYsWleocDocsy1KT\nJk2q1B87dkySNHr06NraAC5LAwcObOgWgAt27NgxXXPNNVXWaw0MLy8v2e32Smt2u13e3t7V1v46\nSOx2u7y8vKoNC0kKDg7WsmXL1KZNGzVt2rTWgwAA1B2n06ljx44pODi42udrDYzOnTvL6XQqNzdX\nnTt3liTt3r1b/v7+VWoDAgK0e/duhYSEVNSdf2H815o3by6bzWZyHACAelDdmcU5tV709vLyUkxM\njFJSUuRwOLRt2zZt2LBBsbGxVWpjY2O1ZMkS5efnKz8/X0uWLNGwYcMurXsAQKPQxDL4okRhYaGe\neeYZffPNN/L19VVCQoLuvPNOZWZmauLEicrKypL0f9/DWLlypSRpxIgReuqpp2p8SwoAcPkwCgwA\nALg1CADACIFxnrNnz+qZZ55RVFSUwsLCNHToUG3atKni+TNnzigpKUkREREKDw+v9SPB69at0623\n3qrQ0FANGjRImZmZdX0Il8yVM8jLy9PEiRN14403qm/fvkpOTlZZWVl9HMYl+a0ZrFmzRmFhYRU/\nPXv2VGBgoHbu3FntvkxvrdPYuGoGtf0+NWau/D04Jzc3Vz169NCTTz5ZH4fgWhYqKSoqslJSUqyD\nBw9aTqfT+vLLL63Q0FDr4MGDlmVZVkJCgvX4449bx48ft8rKyqwffvihxn19/fXX1oABA6ysrCzL\n6XRaR44csY4cOVJfh3LRXDmDCRMmWNOnT7eKi4uto0ePWnfccYe1dOnS+jqUi1bbDH5t1apV1sCB\nA63y8vJq9/XEE09YU6dOtex2u7V161brhhtusPbu3VvXh3DJXDWDC9lPY+PK34Nz4uPjrZEjR1oJ\nCQl11XadITAM3HHHHdann35q7d+/3woLC7NOnz5ttN0999xjffjhh3XcXf242BkMGTLE2rhxY8Xj\nF1980Xr22Wfrqs06dW4G5xszZow1f/78arcpKiqyrr/+euvf//53xdqTTz5pzZ07t876rEsXM4ML\n2c/l4FJm8PHHH1uPPfaYlZKSclkGBm9J1aKgoEC5ubny9/fXjh07dPXVVyslJUURERG688479dln\nn1W7ndPp1M6dO3Xy5EnFxMSof//+Sk5OVnFxcT0fwaW72BlI0n333ad169bpzJkzys/P1+bNm3XT\nTTfVY/eu8esZ/NqhQ4eUmZlZ7cfMpZpvrZOTk1On/daFi52B6X4uB5cyA7vdrpSUFM2YMaOu26wz\nBMZvKC0t1ZNPPqlhw4apa9euOnLkiPbu3SsfHx9t3rxZzz77rGbMmKH9+/dX2bagoEClpaX69NNP\ntWzZMqWlpWnXrl1auHBhAxzJxbuUGUhSr169lJOTo/DwcPXv31/BwcEaNGhQPR/FpTl/Br+WlpYm\nm82mTp06VbvthdxapzG7lBmY7qexu9QZzJs3T3FxcWrfvn1dt1pnCIwalJeX6+mnn5a7u7ueffZZ\nSb98M93d3V0PP/ywPDw81KtXL0VEROjrr7+usn3z5s0lSWPHjlXbtm3l5+en+Pj4y+Zin3TpMygv\nL9f48eMVExOj7du369tvv9WpU6c0d+7c+j6Ui1bdDH4tPT1dQ4cOrXH7C7m1TmN1qTMw3U9jdqkz\nyM7O1r/+9S898MADddhl3SMwqmFZlmbNmqWCggLNnz9f7u7ukqTAwEDjfbRs2VLt2rW7bL+06IoZ\nFBYW6ueff9aYMWPk4eGhVq1aKS4uTl999VVdte1SNc3gnG3btuno0aO65ZZbatzHr2+tc05Nt9Zp\njFwxA5P9NGaumEFGRoYOHTqkqKgo9e3bV3/729+0fv36y+5OGARGNRITE7V//369+eabFWcKkmSz\n2dS+fXu99dZbKisr07Zt25SRkaF+/fpVu5/hw4fr3Xff1fHjx3Xq1CktXbpUAwYMqKejuDSumIGf\nn586duyo5cuXq6ysTP/973+1evXqCwqdhlTTDM5JS0vT4MGDK92h+XwXcmudxsgVMzDZT2Pmihnc\nc889+vzzz5WWlqa0tDTde++9GjBggN555526bN31GvKKe2OUl5dndevWzQoODrZCQ0MrftLT0y3L\nsqy9e/dad999t9WzZ0/r1ltvtdavX1+x7cKFC63x48dXPD579qyVmJhohYeHW5GRkdZzzz1nFRcX\n1/sxXShXzmDXrl3WmDFjLJvNZvXq1cuaMmWKVVBQUO/HdKFqm0FxcbEVHh5uffPNN1W2PX8GJ0+e\ntB5++GGrZ8+e1s0332ytWbOm3o7jUrhqBrXtpzFz5e/Br12un5Li1iAAACO8JQUAMEJgAACMEBgA\nACMEBgDACIEBADBCYAAAjBAYAAAjBAYAwAiBAQAw8v8ALHF/MO+A4gQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11f965b90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 'Dimension reduction - PCA'\n",
    "# print('\\n\\n')\n",
    "# print('PCA for dimensionality reduction')\n",
    "# print('\\n')\n",
    "# pcaref = PCA(n_components=29).fit(data_train)\n",
    "# varref = pcaref.explained_variance_\n",
    "# plt.figure(1)\n",
    "# plt.stem(varref)\n",
    "# refline = np.full([29,1],0.1*varref[0])\n",
    "# plt.plot(refline,'g--')\n",
    "# plt.text(20,0.12*varref[0],'10% of highest variance')\n",
    "# plt.title('Variance of each component')\n",
    "# for i in range (0,29):\n",
    "#     if(varref[i] < 0.1*varref[0]): #Atleast greater than 10% of max variance\n",
    "#         ncomp = i\n",
    "#         break\n",
    "    \n",
    "# #ncomp = 29\n",
    "# pca = PCA(n_components=ncomp).fit(data_train)\n",
    "# datan_train_red = pca.transform(data_train)\n",
    "# datan_test_red = pca.transform(data_test)\n",
    "\n",
    "# 'The features selected are  '\n",
    "# features = np.zeros([1,ncomp])\n",
    "# corr = pca.components_\n",
    "# for f in range (0,ncomp):\n",
    "#     features[0,f] = np.argmax((corr[f,:]))\n",
    "# print('The features selected in order of importance are - ',features)\n",
    "# plt.figure(2)\n",
    "# plt.hist(features,bins=29)\n",
    "# plt.title('Features selected')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
